mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Parallelize get-strings

Browse source
This commit is contained in:
Fedor Indutny 2025-03-27 20:32:24 -07:00 • committed by GitHub
commit ecc8eb6ddd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 73 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -272,7 +272,6 @@
"@types/semver": "7.5.8",
"@types/sinon": "17.0.3",
"@types/split2": "4.2.3",
"@types/unzipper": "0.10.10",
"@types/uuid": "10.0.0",
"@types/websocket": "1.0.0",
"@types/write-file-atomic": "4.0.3",
@ -340,7 +339,6 @@
"terser-webpack-plugin": "5.3.10",
"ts-node": "10.9.2",
"typescript": "5.6.3",
"unzipper": "0.12.3",
"wait-on": "8.0.1",
"webpack": "5.96.1",
"webpack-cli": "5.1.4",

31
pnpm-lock.yaml generated
View file

@ -584,9 +584,6 @@ importers:
'@types/split2':
specifier: 4.2.3
version: 4.2.3
'@types/unzipper':
specifier: 0.10.10
version: 0.10.10
'@types/uuid':
specifier: 10.0.0
version: 10.0.0
@ -788,9 +785,6 @@ importers:
typescript:
specifier: 5.6.3
version: 5.6.3
unzipper:
specifier: 0.12.3
version: 0.12.3
wait-on:
specifier: 8.0.1
version: 8.0.1(debug@4.3.7)
@ -3194,9 +3188,6 @@ packages:
'@types/unist@2.0.11':
resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==}
'@types/unzipper@0.10.10':
resolution: {integrity: sha512-jKJdNxhmCHTZsaKW5x0qjn6rB+gHk0w5VFbEKsw84i+RJqXZyfTmGnpjDcKqzMpjz7VVLsUBMtO5T3mVidpt0g==}
'@types/use-sync-external-store@0.0.3':
resolution: {integrity: sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==}
@ -4760,9 +4751,6 @@ packages:
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
engines: {node: '>= 0.4'}
duplexer2@0.1.4:
resolution: {integrity: sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==}
duplexer3@0.1.5:
resolution: {integrity: sha512-1A8za6ws41LQgv9HrE/66jyC5yuSjQ3L/KOpFtoBilsAK2iA2wuS5rTt1OCzIvtS2V7nVmedsUU+DGRcjBmOYA==}
@ -9326,9 +9314,6 @@ packages:
resolution: {integrity: sha512-N0XH6lqDtFH84JxptQoZYmloF4nzrQqqrAymNj+/gW60AO2AZgOcf4O/nUXJcYfyQkqvMo9lSupBZmmgvuVXlw==}
engines: {node: '>=4'}
unzipper@0.12.3:
resolution: {integrity: sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==}
upath@2.0.1:
resolution: {integrity: sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==}
engines: {node: '>=4'}
@ -13112,10 +13097,6 @@ snapshots:
'@types/unist@2.0.11': {}
'@types/unzipper@0.10.10':
dependencies:
'@types/node': 20.17.6
'@types/use-sync-external-store@0.0.3': {}
'@types/uuid@10.0.0': {}
@ -14897,10 +14878,6 @@ snapshots:
es-errors: 1.3.0
gopd: 1.2.0
duplexer2@0.1.4:
dependencies:
readable-stream: 2.3.8
duplexer3@0.1.5: {}
duplexify@4.1.3:
@ -20500,14 +20477,6 @@ snapshots:
unzip-response@2.0.1: {}
unzipper@0.12.3:
dependencies:
bluebird: 3.7.2
duplexer2: 0.1.4
fs-extra: 11.2.0
graceful-fs: 4.2.11
node-int64: 0.4.0
upath@2.0.1: {}
update-browserslist-db@1.1.2(browserslist@4.24.4):

80
ts/scripts/get-strings.ts
View file

@ -3,10 +3,10 @@
import { rm, mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { Readable } from 'node:stream';
import fastGlob from 'fast-glob';
import unzipper from 'unzipper';
import prettier from 'prettier';
import pMap from 'p-map';
import z from 'zod';
import { authenticate, API_BASE, PROJECT_ID } from '../util/smartling';
@ -30,6 +30,19 @@ const RENAMES = new Map([
['sr-YR', 'sr'],
]);
const StatusSchema = z.object({
response: z.object({
code: z.literal('SUCCESS'),
data: z.object({
items: z
.object({
localeId: z.string(),
})
.array(),
}),
}),
});
async function main() {
if (!SMARTLING_USER) {
console.error('Need to set SMARTLING_USER environment variable!');
@ -46,24 +59,28 @@ async function main() {
userSecret: SMARTLING_SECRET,
});
const zipURL = new URL(
`./files-api/v2/projects/${PROJECT_ID}/locales/all/file/zip`,
const statusURL = new URL(
`./files-api/v2/projects/${PROJECT_ID}/file/status`,
API_BASE
);
zipURL.searchParams.set('fileUri', '_locales/en/messages.json');
zipURL.searchParams.set('retrievalType', 'published');
zipURL.searchParams.set('includeOriginalStrings', 'true');
statusURL.searchParams.set('fileUri', '_locales/en/messages.json');
const fileRes = await fetch(zipURL, {
console.log('Getting list of locales...');
const statusRes = await fetch(statusURL, {
headers,
});
if (!fileRes.ok) {
throw new Error('Failed to fetch the file');
if (!statusRes.ok) {
throw new Error('Failed to fetch the status');
}
if (!fileRes.body) {
if (!statusRes.body) {
throw new Error('Missing body');
}
const {
response: {
data: { items: locales },
},
} = StatusSchema.parse(await statusRes.json());
console.log('Cleaning _locales directory...');
const dirEntries = await fastGlob(['_locales/*', '!_locales/en'], {
@ -79,19 +96,32 @@ async function main() {
const prettierConfig = await prettier.resolveConfig('_locales');
const zip = Readable.from(
fileRes.body as unknown as AsyncIterable<Uint8Array>
).pipe(unzipper.Parse({ forceStream: true }));
for await (const entry of zip) {
if (entry.type !== 'File') {
entry.autodrain();
continue;
await pMap(
locales,
async ({ localeId }) => {
const fileURL = new URL(
`./files-api/v2/projects/${PROJECT_ID}/` +
`locales/${encodeURIComponent(localeId)}/file`,
API_BASE
);
fileURL.searchParams.set('fileUri', '_locales/en/messages.json');
fileURL.searchParams.set('retrievalType', 'published');
fileURL.searchParams.set('includeOriginalStrings', 'true');
const fileRes = await fetch(fileURL, {
headers,
});
if (!fileRes.ok) {
throw new Error('Failed to fetch the file');
}
if (!fileRes.body) {
throw new Error('Missing body');
}
let [locale] = entry.path.split(/[\\/]/, 1);
locale = RENAMES.get(locale) ?? locale;
const targetLocale = RENAMES.get(localeId) ?? localeId;
const targetDir = path.join('_locales', targetLocale);
const targetDir = path.join('_locales', locale);
try {
await mkdir(targetDir);
} catch (error) {
@ -99,8 +129,8 @@ async function main() {
}
const targetFile = path.join(targetDir, 'messages.json');
console.log('Writing', locale);
const json = JSON.parse((await entry.buffer()).toString());
console.log('Writing', targetLocale);
const json = await fileRes.json();
for (const value of Object.values(json)) {
const typedValue = value as { description?: string };
delete typedValue.description;
@ -111,7 +141,9 @@ async function main() {
filepath: targetFile,
});
await writeFile(targetFile, output);
}
},
{ concurrency: 20 }
);
}
main().catch(err => {