support icu messageformat for translations

build/intl-linter/linter.ts

@@ -0,0 +1,177 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { parse as parseIcuMessage } from '@formatjs/icu-messageformat-parser';
+import type {
+  MessageFormatElement,
+  Location,
+} from '@formatjs/icu-messageformat-parser';
+import parseJsonToAst from 'json-to-ast';
+import { readFile } from 'fs/promises';
+import { join as pathJoin, relative as pathRelative } from 'path';
+import chalk from 'chalk';
+import { deepEqual } from 'assert';
+import type { Rule } from './utils/rule';
+
+import icuPrefix from './rules/icuPrefix';
+import onePlural from './rules/onePlural';
+import noLegacyVariables from './rules/noLegacyVariables';
+import noNestedChoice from './rules/noNestedChoice';
+import noOffset from './rules/noOffset';
+import noOrdinal from './rules/noOrdinal';
+
+const RULES = [
+  icuPrefix,
+  noLegacyVariables,
+  noNestedChoice,
+  noOffset,
+  noOrdinal,
+  onePlural,
+];
+
+type Test = {
+  messageformat: string;
+  expectErrors: Array<string>;
+};
+
+const tests: Record<string, Test> = {
+  err1: {
+    messageformat: '{a, plural, other {a}} {b, plural, other {b}}',
+    expectErrors: ['onePlural'],
+  },
+  err2: {
+    messageformat: '{a, plural, other {{b, plural, other {b}}}}',
+    expectErrors: ['noNestedChoice', 'onePlural'],
+  },
+  err3: {
+    messageformat: '{a, select, other {{b, select, other {b}}}}',
+    expectErrors: ['noNestedChoice'],
+  },
+  err4: {
+    messageformat: '{a, plural, offset:1 other {a}}',
+    expectErrors: ['noOffset'],
+  },
+  err5: {
+    messageformat: '{a, selectordinal, other {a}}',
+    expectErrors: ['noOrdinal'],
+  },
+  err6: {
+    messageformat: '$a$',
+    expectErrors: ['noLegacyVariables'],
+  },
+};
+
+type Report = {
+  id: string;
+  message: string;
+  location: Location | void;
+};
+
+function lintMessage(
+  messageId: string,
+  elements: Array<MessageFormatElement>,
+  rules: Array<Rule>
+) {
+  const reports: Array<Report> = [];
+  for (const rule of rules) {
+    rule.run(elements, {
+      messageId,
+      report(message, location) {
+        reports.push({ id: rule.id, message, location });
+      },
+    });
+  }
+  return reports;
+}
+
+function assert(condition: unknown, message: string): asserts condition {
+  if (!condition) {
+    throw new Error(message);
+  }
+}
+
+async function lintMessages() {
+  const repoRoot = pathJoin(__dirname, '..', '..');
+  const filePath = pathJoin(repoRoot, '_locales/en/messages.json');
+  const relativePath = pathRelative(repoRoot, filePath);
+  let file = await readFile(filePath, 'utf-8');
+
+  if (process.argv.includes('--test')) {
+    file = JSON.stringify(tests);
+  }
+
+  const jsonAst = parseJsonToAst(file);
+
+  assert(jsonAst.type === 'Object', 'Expected an object');
+  for (const topProp of jsonAst.children) {
+    if (topProp.key.value === 'smartling') {
+      continue;
+    }
+
+    const messageId = topProp.key.value;
+
+    assert(topProp.value.type === 'Object', 'Expected an object');
+
+    const icuMessageProp = topProp.value.children.find(messageProp => {
+      return messageProp.key.value === 'messageformat';
+    });
+    if (icuMessageProp == null) {
+      continue;
+    }
+
+    const icuMesssageLiteral = icuMessageProp.value;
+    assert(
+      icuMesssageLiteral.type === 'Literal' &&
+        typeof icuMesssageLiteral.value === 'string',
+      'Expected a string'
+    );
+
+    const icuMessage: string = icuMesssageLiteral.value;
+
+    const ast = parseIcuMessage(icuMessage, {
+      captureLocation: true,
+      shouldParseSkeletons: true,
+      requiresOtherClause: true,
+    });
+
+    const reports = lintMessage(messageId, ast, RULES);
+    const key = topProp.key.value;
+
+    if (process.argv.includes('--test')) {
+      const test = tests[key];
+      const actualErrors = reports.map(report => report.id);
+      deepEqual(actualErrors, test.expectErrors);
+      continue;
+    }
+
+    for (const report of reports) {
+      let loc = '';
+
+      if (report.location != null && icuMesssageLiteral.loc != null) {
+        const line =
+          icuMesssageLiteral.loc.start.line + (report.location.start.line - 1);
+        const column =
+          icuMesssageLiteral.loc.start.column + report.location.start.column;
+        loc = `:${line}:${column}`;
+      } else if (icuMesssageLiteral.loc != null) {
+        const { line, column } = icuMesssageLiteral.loc.start;
+        loc = `:${line}:${column}`;
+      }
+
+      // eslint-disable-next-line no-console
+      console.error(
+        chalk`{bold.cyan ${relativePath}${loc}} ${report.message} {magenta ({underline ${report.id}})}`
+      );
+      // eslint-disable-next-line no-console
+      console.error(chalk`  {dim in ${key} is "}{red ${icuMessage}}{dim "}`);
+      // eslint-disable-next-line no-console
+      console.error();
+    }
+  }
+}
+
+lintMessages().catch(error => {
+  // eslint-disable-next-line no-console
+  console.error(error);
+  process.exit(1);
+});

build/intl-linter/rules/icuPrefix.ts

@@ -0,0 +1,11 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { rule } from '../utils/rule';
+
+export default rule('icuPrefix', context => {
+  if (!context.messageId.startsWith('icu:')) {
+    context.report('ICU message IDs must start with "icu:"');
+  }
+  return {};
+});

build/intl-linter/rules/noLegacyVariables.ts

@@ -0,0 +1,17 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { rule } from '../utils/rule';
+
+export default rule('noLegacyVariables', context => {
+  return {
+    enterLiteral(element) {
+      if (/(\$.+?\$)/.test(element.value)) {
+        context.report(
+          'String must not contain legacy $variables$',
+          element.location
+        );
+      }
+    },
+  };
+});

build/intl-linter/rules/noNestedChoice.ts

@@ -0,0 +1,35 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Element } from '../utils/rule';
+import { rule } from '../utils/rule';
+
+export default rule('noNestedChoice', context => {
+  let insideChoice = false;
+
+  function check(element: Element) {
+    if (insideChoice) {
+      context.report(
+        'Nested {select}/{plural} is not supported by Smartling',
+        element.location
+      );
+    }
+  }
+
+  return {
+    enterSelect(element) {
+      check(element);
+      insideChoice = true;
+    },
+    exitSelect() {
+      insideChoice = false;
+    },
+    enterPlural(element) {
+      check(element);
+      insideChoice = true;
+    },
+    exitPlural() {
+      insideChoice = false;
+    },
+  };
+});

build/intl-linter/rules/noOffset.ts

@@ -0,0 +1,17 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { rule } from '../utils/rule';
+
+export default rule('noOffset', context => {
+  return {
+    enterPlural(element) {
+      if (element.offset !== 0) {
+        context.report(
+          '{plural} with offset is not supported by Smartling',
+          element.location
+        );
+      }
+    },
+  };
+});

build/intl-linter/rules/noOrdinal.ts

@@ -0,0 +1,17 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { rule } from '../utils/rule';
+
+export default rule('noOrdinal', context => {
+  return {
+    enterPlural(element) {
+      if (element.pluralType === 'ordinal') {
+        context.report(
+          '{selectordinal} is not supported by Smartling',
+          element.location
+        );
+      }
+    },
+  };
+});

build/intl-linter/rules/onePlural.ts

@@ -0,0 +1,19 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { rule } from '../utils/rule';
+
+export default rule('onePlural', context => {
+  let plurals = 0;
+  return {
+    enterPlural(element) {
+      plurals += 1;
+      if (plurals > 1) {
+        context.report(
+          'Multiple {plural} is not supported by Smartling',
+          element.location
+        );
+      }
+    },
+  };
+});

build/intl-linter/utils/rule.ts

@@ -0,0 +1,33 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { MessageFormatElement } from '@formatjs/icu-messageformat-parser';
+import { Location } from '@formatjs/icu-messageformat-parser';
+import type { Visitor } from './traverse';
+import { traverse } from './traverse';
+
+export type Element = MessageFormatElement;
+export { Location };
+
+export type Context = {
+  messageId: string;
+  report(message: string, location: Location | void): void;
+};
+
+export type RuleFactory = {
+  (context: Context): Visitor;
+};
+
+export type Rule = {
+  id: string;
+  run(elements: Array<MessageFormatElement>, context: Context): void;
+};
+
+export function rule(id: string, ruleFactory: RuleFactory): Rule {
+  return {
+    id,
+    run(elements, context) {
+      traverse(elements, ruleFactory(context));
+    },
+  };
+}

build/intl-linter/utils/traverse.ts

@@ -0,0 +1,90 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type {
+  MessageFormatElement,
+  LiteralElement,
+  ArgumentElement,
+  NumberElement,
+  DateElement,
+  TimeElement,
+  SelectElement,
+  PluralElement,
+  PoundElement,
+  TagElement,
+} from '@formatjs/icu-messageformat-parser';
+import { TYPE } from '@formatjs/icu-messageformat-parser';
+
+export type VisitorMethod<T extends MessageFormatElement> = (
+  element: T
+) => void;
+
+export type Visitor = {
+  enterLiteral?: VisitorMethod<LiteralElement>;
+  exitLiteral?: VisitorMethod<LiteralElement>;
+  enterArgument?: VisitorMethod<ArgumentElement>;
+  exitArgument?: VisitorMethod<ArgumentElement>;
+  enterNumber?: VisitorMethod<NumberElement>;
+  exitNumber?: VisitorMethod<NumberElement>;
+  enterDate?: VisitorMethod<DateElement>;
+  exitDate?: VisitorMethod<DateElement>;
+  enterTime?: VisitorMethod<TimeElement>;
+  exitTime?: VisitorMethod<TimeElement>;
+  enterSelect?: VisitorMethod<SelectElement>;
+  exitSelect?: VisitorMethod<SelectElement>;
+  enterPlural?: VisitorMethod<PluralElement>;
+  exitPlural?: VisitorMethod<PluralElement>;
+  enterPound?: VisitorMethod<PoundElement>;
+  exitPound?: VisitorMethod<PoundElement>;
+  enterTag?: VisitorMethod<TagElement>;
+  exitTag?: VisitorMethod<TagElement>;
+};
+
+export function traverse(
+  elements: Array<MessageFormatElement>,
+  visitor: Visitor
+): void {
+  for (const element of elements) {
+    if (element.type === TYPE.literal) {
+      visitor.enterLiteral?.(element);
+      visitor.exitLiteral?.(element);
+    } else if (element.type === TYPE.argument) {
+      visitor.enterArgument?.(element);
+      visitor.exitArgument?.(element);
+    } else if (element.type === TYPE.number) {
+      visitor.enterNumber?.(element);
+      visitor.exitNumber?.(element);
+    } else if (element.type === TYPE.date) {
+      visitor.enterDate?.(element);
+      visitor.exitDate?.(element);
+    } else if (element.type === TYPE.time) {
+      visitor.enterTime?.(element);
+      visitor.exitTime?.(element);
+    } else if (element.type === TYPE.select) {
+      visitor.enterSelect?.(element);
+      for (const node of Object.values(element.options)) {
+        traverse(node.value, visitor);
+      }
+      visitor.exitSelect?.(element);
+    } else if (element.type === TYPE.plural) {
+      visitor.enterPlural?.(element);
+      for (const node of Object.values(element.options)) {
+        traverse(node.value, visitor);
+      }
+      visitor.exitPlural?.(element);
+    } else if (element.type === TYPE.pound) {
+      visitor.enterPound?.(element);
+      visitor.exitPound?.(element);
+    } else if (element.type === TYPE.tag) {
+      visitor.enterTag?.(element);
+      traverse(element.children, visitor);
+      visitor.exitTag?.(element);
+    } else {
+      unreachable(element);
+    }
+  }
+}
+
+function unreachable(x: never): never {
+  throw new Error(`unreachable: ${x}`);
+}