Parallelize get-strings

package.json

@@ -272,7 +272,6 @@
     "@types/semver": "7.5.8",
     "@types/sinon": "17.0.3",
     "@types/split2": "4.2.3",
-    "@types/unzipper": "0.10.10",
     "@types/uuid": "10.0.0",
     "@types/websocket": "1.0.0",
     "@types/write-file-atomic": "4.0.3",
@@ -340,7 +339,6 @@
     "terser-webpack-plugin": "5.3.10",
     "ts-node": "10.9.2",
     "typescript": "5.6.3",
-    "unzipper": "0.12.3",
     "wait-on": "8.0.1",
     "webpack": "5.96.1",
     "webpack-cli": "5.1.4",

pnpm-lock.yaml

@@ -584,9 +584,6 @@ importers:
       '@types/split2':
         specifier: 4.2.3
         version: 4.2.3
-      '@types/unzipper':
-        specifier: 0.10.10
-        version: 0.10.10
       '@types/uuid':
         specifier: 10.0.0
         version: 10.0.0
@@ -788,9 +785,6 @@ importers:
       typescript:
         specifier: 5.6.3
         version: 5.6.3
-      unzipper:
-        specifier: 0.12.3
-        version: 0.12.3
       wait-on:
         specifier: 8.0.1
         version: 8.0.1(debug@4.3.7)
@@ -3194,9 +3188,6 @@ packages:
   '@types/unist@2.0.11':
     resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
 
-  '@types/unzipper@0.10.10':
-    resolution: {integrity: sha512-jKJdNxhmCHTZsaKW5x0qjn6rB+gHk0w5VFbEKsw84i+RJqXZyfTmGnpjDcKqzMpjz7VVLsUBMtO5T3mVidpt0g==}
-
   '@types/use-sync-external-store@0.0.3':
     resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==}
 
@@ -4760,9 +4751,6 @@ packages:
     resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
     engines: {node: '>= 0.4'}
 
-  duplexer2@0.1.4:
-    resolution: {integrity: sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==}
-
   duplexer3@0.1.5:
     resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==}
 
@@ -9326,9 +9314,6 @@ packages:
     resolution: {integrity: sha512-N0XH6lqDtFH84JxptQoZYmloF4nzrQqqrAymNj+/gW60AO2AZgOcf4O/nUXJcYfyQkqvMo9lSupBZmmgvuVXlw==}
     engines: {node: '>=4'}
 
-  unzipper@0.12.3:
-    resolution: {integrity: sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==}
-
   upath@2.0.1:
     resolution: {integrity: sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==}
     engines: {node: '>=4'}
@@ -13112,10 +13097,6 @@ snapshots:
 
   '@types/unist@2.0.11': {}
 
-  '@types/unzipper@0.10.10':
-    dependencies:
-      '@types/node': 20.17.6
-
   '@types/use-sync-external-store@0.0.3': {}
 
   '@types/uuid@10.0.0': {}
@@ -14897,10 +14878,6 @@ snapshots:
       es-errors: 1.3.0
       gopd: 1.2.0
 
-  duplexer2@0.1.4:
-    dependencies:
-      readable-stream: 2.3.8
-
   duplexer3@0.1.5: {}
 
   duplexify@4.1.3:
@@ -20500,14 +20477,6 @@ snapshots:
 
   unzip-response@2.0.1: {}
 
-  unzipper@0.12.3:
-    dependencies:
-      bluebird: 3.7.2
-      duplexer2: 0.1.4
-      fs-extra: 11.2.0
-      graceful-fs: 4.2.11
-      node-int64: 0.4.0
-
   upath@2.0.1: {}
 
   update-browserslist-db@1.1.2(browserslist@4.24.4):

ts/scripts/get-strings.ts

@@ -3,10 +3,10 @@
 
 import { rm, mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import { Readable } from 'node:stream';
 import fastGlob from 'fast-glob';
-import unzipper from 'unzipper';
 import prettier from 'prettier';
+import pMap from 'p-map';
+import z from 'zod';
 
 import { authenticate, API_BASE, PROJECT_ID } from '../util/smartling';
 
@@ -30,6 +30,19 @@ const RENAMES = new Map([
   ['sr-YR', 'sr'],
 ]);
 
+const StatusSchema = z.object({
+  response: z.object({
+    code: z.literal('SUCCESS'),
+    data: z.object({
+      items: z
+        .object({
+          localeId: z.string(),
+        })
+        .array(),
+    }),
+  }),
+});
+
 async function main() {
   if (!SMARTLING_USER) {
     console.error('Need to set SMARTLING_USER environment variable!');
@@ -46,24 +59,28 @@ async function main() {
     userSecret: SMARTLING_SECRET,
   });
 
-  const zipURL = new URL(
+  const statusURL = new URL(
-    `./files-api/v2/projects/${PROJECT_ID}/locales/all/file/zip`,
+    `./files-api/v2/projects/${PROJECT_ID}/file/status`,
     API_BASE
   );
-  zipURL.searchParams.set('fileUri', '_locales/en/messages.json');
+  statusURL.searchParams.set('fileUri', '_locales/en/messages.json');
-  zipURL.searchParams.set('retrievalType', 'published');
-  zipURL.searchParams.set('includeOriginalStrings', 'true');
 
-  const fileRes = await fetch(zipURL, {
+  console.log('Getting list of locales...');
+  const statusRes = await fetch(statusURL, {
     headers,
   });
 
-  if (!fileRes.ok) {
+  if (!statusRes.ok) {
-    throw new Error('Failed to fetch the file');
+    throw new Error('Failed to fetch the status');
   }
-  if (!fileRes.body) {
+  if (!statusRes.body) {
     throw new Error('Missing body');
   }
+  const {
+    response: {
+      data: { items: locales },
+    },
+  } = StatusSchema.parse(await statusRes.json());
 
   console.log('Cleaning _locales directory...');
   const dirEntries = await fastGlob(['_locales/*', '!_locales/en'], {
@@ -79,39 +96,54 @@ async function main() {
 
   const prettierConfig = await prettier.resolveConfig('_locales');
 
-  const zip = Readable.from(
+  await pMap(
-    fileRes.body as unknown as AsyncIterable<Uint8Array>
+    locales,
-  ).pipe(unzipper.Parse({ forceStream: true }));
+    async ({ localeId }) => {
-  for await (const entry of zip) {
+      const fileURL = new URL(
-    if (entry.type !== 'File') {
+        `./files-api/v2/projects/${PROJECT_ID}/` +
-      entry.autodrain();
+          `locales/${encodeURIComponent(localeId)}/file`,
-      continue;
+        API_BASE
-    }
+      );
+      fileURL.searchParams.set('fileUri', '_locales/en/messages.json');
+      fileURL.searchParams.set('retrievalType', 'published');
+      fileURL.searchParams.set('includeOriginalStrings', 'true');
 
-    let [locale] = entry.path.split(/[\\/]/, 1);
+      const fileRes = await fetch(fileURL, {
-    locale = RENAMES.get(locale) ?? locale;
+        headers,
+      });
 
-    const targetDir = path.join('_locales', locale);
+      if (!fileRes.ok) {
-    try {
+        throw new Error('Failed to fetch the file');
-      await mkdir(targetDir);
+      }
-    } catch (error) {
+      if (!fileRes.body) {
-      console.error(error);
+        throw new Error('Missing body');
-    }
+      }
 
-    const targetFile = path.join(targetDir, 'messages.json');
+      const targetLocale = RENAMES.get(localeId) ?? localeId;
-    console.log('Writing', locale);
+      const targetDir = path.join('_locales', targetLocale);
-    const json = JSON.parse((await entry.buffer()).toString());
+
-    for (const value of Object.values(json)) {
+      try {
-      const typedValue = value as { description?: string };
+        await mkdir(targetDir);
-      delete typedValue.description;
+      } catch (error) {
-    }
+        console.error(error);
-    delete json.smartling;
+      }
-    const output = await prettier.format(JSON.stringify(json, null, 2), {
+
-      ...prettierConfig,
+      const targetFile = path.join(targetDir, 'messages.json');
-      filepath: targetFile,
+      console.log('Writing', targetLocale);
-    });
+      const json = await fileRes.json();
-    await writeFile(targetFile, output);
+      for (const value of Object.values(json)) {
-  }
+        const typedValue = value as { description?: string };
+        delete typedValue.description;
+      }
+      delete json.smartling;
+      const output = await prettier.format(JSON.stringify(json, null, 2), {
+        ...prettierConfig,
+        filepath: targetFile,
+      });
+      await writeFile(targetFile, output);
+    },
+    { concurrency: 20 }
+  );
 }
 
 main().catch(err => {