diff --git a/package.json b/package.json index 9a0d6de48b6..25748e04ba5 100644 --- a/package.json +++ b/package.json @@ -272,7 +272,6 @@ "@types/semver": "7.5.8", "@types/sinon": "17.0.3", "@types/split2": "4.2.3", - "@types/unzipper": "0.10.10", "@types/uuid": "10.0.0", "@types/websocket": "1.0.0", "@types/write-file-atomic": "4.0.3", @@ -340,7 +339,6 @@ "terser-webpack-plugin": "5.3.10", "ts-node": "10.9.2", "typescript": "5.6.3", - "unzipper": "0.12.3", "wait-on": "8.0.1", "webpack": "5.96.1", "webpack-cli": "5.1.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 76fe1ecc5ad..2cfce04b07a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -584,9 +584,6 @@ importers: '@types/split2': specifier: 4.2.3 version: 4.2.3 - '@types/unzipper': - specifier: 0.10.10 - version: 0.10.10 '@types/uuid': specifier: 10.0.0 version: 10.0.0 @@ -788,9 +785,6 @@ importers: typescript: specifier: 5.6.3 version: 5.6.3 - unzipper: - specifier: 0.12.3 - version: 0.12.3 wait-on: specifier: 8.0.1 version: 8.0.1(debug@4.3.7) @@ -3194,9 +3188,6 @@ packages: '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} - '@types/unzipper@0.10.10': - resolution: {integrity: sha512-jKJdNxhmCHTZsaKW5x0qjn6rB+gHk0w5VFbEKsw84i+RJqXZyfTmGnpjDcKqzMpjz7VVLsUBMtO5T3mVidpt0g==} - '@types/use-sync-external-store@0.0.3': resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==} @@ -4760,9 +4751,6 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} - duplexer2@0.1.4: - resolution: {integrity: sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==} - duplexer3@0.1.5: resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==} @@ -9326,9 +9314,6 @@ packages: resolution: {integrity: sha512-N0XH6lqDtFH84JxptQoZYmloF4nzrQqqrAymNj+/gW60AO2AZgOcf4O/nUXJcYfyQkqvMo9lSupBZmmgvuVXlw==} engines: {node: '>=4'} - unzipper@0.12.3: - resolution: {integrity: sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==} - upath@2.0.1: resolution: {integrity: sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==} engines: {node: '>=4'} @@ -13112,10 +13097,6 @@ snapshots: '@types/unist@2.0.11': {} - '@types/unzipper@0.10.10': - dependencies: - '@types/node': 20.17.6 - '@types/use-sync-external-store@0.0.3': {} '@types/uuid@10.0.0': {} @@ -14897,10 +14878,6 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 - duplexer2@0.1.4: - dependencies: - readable-stream: 2.3.8 - duplexer3@0.1.5: {} duplexify@4.1.3: @@ -20500,14 +20477,6 @@ snapshots: unzip-response@2.0.1: {} - unzipper@0.12.3: - dependencies: - bluebird: 3.7.2 - duplexer2: 0.1.4 - fs-extra: 11.2.0 - graceful-fs: 4.2.11 - node-int64: 0.4.0 - upath@2.0.1: {} update-browserslist-db@1.1.2(browserslist@4.24.4): diff --git a/ts/scripts/get-strings.ts b/ts/scripts/get-strings.ts index e2305d82fc5..756aaef9c00 100644 --- a/ts/scripts/get-strings.ts +++ b/ts/scripts/get-strings.ts @@ -3,10 +3,10 @@ import { rm, mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; -import { Readable } from 'node:stream'; import fastGlob from 'fast-glob'; -import unzipper from 'unzipper'; import prettier from 'prettier'; +import pMap from 'p-map'; +import z from 'zod'; import { authenticate, API_BASE, PROJECT_ID } from '../util/smartling'; @@ -30,6 +30,19 @@ const RENAMES = new Map([ ['sr-YR', 'sr'], ]); +const StatusSchema = z.object({ + response: z.object({ + code: z.literal('SUCCESS'), + data: z.object({ + items: z + .object({ + localeId: z.string(), + }) + .array(), + }), + }), +}); + async function main() { if (!SMARTLING_USER) { console.error('Need to set SMARTLING_USER environment variable!'); @@ -46,24 +59,28 @@ async function main() { userSecret: SMARTLING_SECRET, }); - const zipURL = new URL( - `./files-api/v2/projects/${PROJECT_ID}/locales/all/file/zip`, + const statusURL = new URL( + `./files-api/v2/projects/${PROJECT_ID}/file/status`, API_BASE ); - zipURL.searchParams.set('fileUri', '_locales/en/messages.json'); - zipURL.searchParams.set('retrievalType', 'published'); - zipURL.searchParams.set('includeOriginalStrings', 'true'); + statusURL.searchParams.set('fileUri', '_locales/en/messages.json'); - const fileRes = await fetch(zipURL, { + console.log('Getting list of locales...'); + const statusRes = await fetch(statusURL, { headers, }); - if (!fileRes.ok) { - throw new Error('Failed to fetch the file'); + if (!statusRes.ok) { + throw new Error('Failed to fetch the status'); } - if (!fileRes.body) { + if (!statusRes.body) { throw new Error('Missing body'); } + const { + response: { + data: { items: locales }, + }, + } = StatusSchema.parse(await statusRes.json()); console.log('Cleaning _locales directory...'); const dirEntries = await fastGlob(['_locales/*', '!_locales/en'], { @@ -79,39 +96,54 @@ async function main() { const prettierConfig = await prettier.resolveConfig('_locales'); - const zip = Readable.from( - fileRes.body as unknown as AsyncIterable - ).pipe(unzipper.Parse({ forceStream: true })); - for await (const entry of zip) { - if (entry.type !== 'File') { - entry.autodrain(); - continue; - } + await pMap( + locales, + async ({ localeId }) => { + const fileURL = new URL( + `./files-api/v2/projects/${PROJECT_ID}/` + + `locales/${encodeURIComponent(localeId)}/file`, + API_BASE + ); + fileURL.searchParams.set('fileUri', '_locales/en/messages.json'); + fileURL.searchParams.set('retrievalType', 'published'); + fileURL.searchParams.set('includeOriginalStrings', 'true'); - let [locale] = entry.path.split(/[\\/]/, 1); - locale = RENAMES.get(locale) ?? locale; + const fileRes = await fetch(fileURL, { + headers, + }); - const targetDir = path.join('_locales', locale); - try { - await mkdir(targetDir); - } catch (error) { - console.error(error); - } + if (!fileRes.ok) { + throw new Error('Failed to fetch the file'); + } + if (!fileRes.body) { + throw new Error('Missing body'); + } - const targetFile = path.join(targetDir, 'messages.json'); - console.log('Writing', locale); - const json = JSON.parse((await entry.buffer()).toString()); - for (const value of Object.values(json)) { - const typedValue = value as { description?: string }; - delete typedValue.description; - } - delete json.smartling; - const output = await prettier.format(JSON.stringify(json, null, 2), { - ...prettierConfig, - filepath: targetFile, - }); - await writeFile(targetFile, output); - } + const targetLocale = RENAMES.get(localeId) ?? localeId; + const targetDir = path.join('_locales', targetLocale); + + try { + await mkdir(targetDir); + } catch (error) { + console.error(error); + } + + const targetFile = path.join(targetDir, 'messages.json'); + console.log('Writing', targetLocale); + const json = await fileRes.json(); + for (const value of Object.values(json)) { + const typedValue = value as { description?: string }; + delete typedValue.description; + } + delete json.smartling; + const output = await prettier.format(JSON.stringify(json, null, 2), { + ...prettierConfig, + filepath: targetFile, + }); + await writeFile(targetFile, output); + }, + { concurrency: 20 } + ); } main().catch(err => {