From 1ca121aef56249ada5ed13ad55636cb15801d768 Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Wed, 14 Apr 2021 12:43:11 -0700 Subject: [PATCH] Bundle sql worker with webpack --- .gitignore | 1 + package.json | 21 ++++------ preload.js | 31 +-------------- preload_test.js | 31 +++++++++++++++ ts/sql/main.ts | 18 ++++++--- ts/sql/mainWorkerBindings.ts | 13 +++++++ webpack-preload.config.ts | 74 ++++++++++++++++++++++++++++++++++++ webpack-sql-worker.config.ts | 36 ++++++++++++++++++ webpack.config.ts | 62 +----------------------------- 9 files changed, 177 insertions(+), 110 deletions(-) create mode 100644 preload_test.js create mode 100644 ts/sql/mainWorkerBindings.ts create mode 100644 webpack-preload.config.ts create mode 100644 webpack-sql-worker.config.ts diff --git a/.gitignore b/.gitignore index 27401750c976..8a196c9ba7fa 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ sticker-creator/dist/* /.idea /storybook-static/ preload.bundle.* +ts/sql/mainWorker.bundle.js.LICENSE.txt diff --git a/package.json b/package.json index 923472f1c826..2882466a833e 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,10 @@ "build:dev": "run-s --print-label build:grunt build:typed-scss build:webpack", "build:grunt": "yarn grunt", "build:typed-scss": "tsm sticker-creator", - "build:webpack": "cross-env NODE_ENV=production webpack", + "build:webpack": "run-p build:webpack:sticker-creator build:webpack:preload build:webpack:sql-worker", + "build:webpack:sticker-creator": "cross-env NODE_ENV=production webpack", + "build:webpack:preload": "cross-env NODE_ENV=production webpack -c webpack-preload.config.ts", + "build:webpack:sql-worker": "cross-env NODE_ENV=production webpack -c webpack-sql-worker.config.ts", "build:electron": "electron-builder --config.extraMetadata.environment=$SIGNAL_ENV", "build:release": "cross-env SIGNAL_ENV=production yarn build:electron -- --config.directories.output=release", "build:zip": "node scripts/zip-macos-release.js", @@ -366,18 +369,8 @@ ] }, "asarUnpack": [ - "js/modules/privacy.js", - "ts/environment.js", - "ts/logging/log.js", - "ts/logging/shared.js", - "ts/sql/Server.js", - "ts/sql/mainWorker.js", - "ts/util/assert.js", - "ts/util/combineNames.js", - "ts/util/enum.js", - "ts/util/isNormalNumber.js", - "ts/util/missingCaseError.js", - "ts/util/reallyJsonStringify.js" + "ts/sql/mainWorker.bundle.js", + "node_modules/better-sqlite3/build/Release/better_sqlite3.node" ], "files": [ "package.json", @@ -438,7 +431,7 @@ "!node_modules/sharp/{install,src,vendor/include,vendor/*/include}", "!node_modules/better-sqlite3/deps/*", "!node_modules/better-sqlite3/src/*", - "node_modules/better-sqlite3/build/Release/*.node", + "node_modules/better-sqlite3/build/Release/better_sqlite3.node", "node_modules/libsignal-client/build/*${platform}*.node", "node_modules/ringrtc/build/${platform}/**", "!**/node_modules/ffi-napi/deps", diff --git a/preload.js b/preload.js index cb07a93c3ef6..c91014f6f9e9 100644 --- a/preload.js +++ b/preload.js @@ -688,36 +688,7 @@ try { }); if (config.environment === 'test') { - // This is a hack to let us run TypeScript tests in the renderer process. See the - // code in `test/index.html`. - const pendingDescribeCalls = []; - window.describe = (...args) => { - pendingDescribeCalls.push(args); - }; - - /* eslint-disable global-require, import/no-extraneous-dependencies */ - const fastGlob = require('fast-glob'); - - fastGlob - .sync('./ts/test-{both,electron}/**/*_test.js', { - absolute: true, - cwd: __dirname, - }) - .forEach(require); - - delete window.describe; - - window.test = { - pendingDescribeCalls, - fastGlob, - normalizePath: require('normalize-path'), - fse: require('fs-extra'), - tmp: require('tmp'), - path: require('path'), - basePath: __dirname, - attachmentsPath: window.Signal.Migrations.attachmentsPath, - }; - /* eslint-enable global-require, import/no-extraneous-dependencies */ + require('./preload_test.js'); } } catch (error) { /* eslint-disable no-console */ diff --git a/preload_test.js b/preload_test.js new file mode 100644 index 000000000000..2aca6f16c769 --- /dev/null +++ b/preload_test.js @@ -0,0 +1,31 @@ +/* global window */ + +// This is a hack to let us run TypeScript tests in the renderer process. See the +// code in `test/index.html`. +const pendingDescribeCalls = []; +window.describe = (...args) => { + pendingDescribeCalls.push(args); +}; + +/* eslint-disable global-require, import/no-extraneous-dependencies */ +const fastGlob = require('fast-glob'); + +fastGlob + .sync('./ts/test-{both,electron}/**/*_test.js', { + absolute: true, + cwd: __dirname, + }) + .forEach(require); + +delete window.describe; + +window.test = { + pendingDescribeCalls, + fastGlob, + normalizePath: require('normalize-path'), + fse: require('fs-extra'), + tmp: require('tmp'), + path: require('path'), + basePath: __dirname, + attachmentsPath: window.Signal.Migrations.attachmentsPath, +}; diff --git a/ts/sql/main.ts b/ts/sql/main.ts index 7dd03a1a4a82..a114c9a16ffe 100644 --- a/ts/sql/main.ts +++ b/ts/sql/main.ts @@ -4,6 +4,8 @@ import { join } from 'path'; import { Worker } from 'worker_threads'; +const ASAR_PATTERN = /app\.asar$/; + export type InitializeOptions = { readonly configDir: string; readonly key: string; @@ -56,12 +58,18 @@ export class MainSQL { private onResponse = new Map>(); constructor() { - const appDir = join(__dirname, '..', '..').replace( - /app\.asar$/, - 'app.asar.unpacked' - ); + let appDir = join(__dirname, '..', '..'); + let isBundled = false; - this.worker = new Worker(join(appDir, 'ts', 'sql', 'mainWorker.js')); + if (ASAR_PATTERN.test(appDir)) { + appDir = appDir.replace(ASAR_PATTERN, 'app.asar.unpacked'); + isBundled = true; + } + + const scriptDir = join(appDir, 'ts', 'sql'); + this.worker = new Worker( + join(scriptDir, isBundled ? 'mainWorker.bundle.js' : 'mainWorker.js') + ); this.worker.on('message', (wrappedResponse: WrappedWorkerResponse) => { const { seq, error, response } = wrappedResponse; diff --git a/ts/sql/mainWorkerBindings.ts b/ts/sql/mainWorkerBindings.ts new file mode 100644 index 000000000000..34eae4b8d531 --- /dev/null +++ b/ts/sql/mainWorkerBindings.ts @@ -0,0 +1,13 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +// This is a shim that gets inserted in place of `bindings` npm module when +// building sql worker bundle. +module.exports = (binding: string) => { + if (binding === 'better_sqlite3.node') { + // eslint-disable-next-line global-require, import/no-unresolved + return require('better_sqlite3.node'); + } + + throw new Error(`Unknown binding ${binding}`); +}; diff --git a/webpack-preload.config.ts b/webpack-preload.config.ts new file mode 100644 index 000000000000..ef9235f98fc3 --- /dev/null +++ b/webpack-preload.config.ts @@ -0,0 +1,74 @@ +// Copyright 2019-2020 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { resolve } from 'path'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { Configuration } from 'webpack'; +import TerserPlugin = require('terser-webpack-plugin'); + +const context = __dirname; +const { NODE_ENV: mode = 'development' } = process.env; + +const EXTERNAL_MODULE = new Set([ + 'backbone', + 'better-sqlite3', + 'ffi-napi', + 'fs-xattr', + 'fsevents', + 'got', + 'jquery', + 'libsignal-client', + 'node-fetch', + 'node-sass', + 'pino', + 'proxy-agent', + 'ref-array-napi', + 'ref-napi', + 'ringrtc', + 'sharp', + 'websocket', + 'zkgroup', + + // Uses fast-glob and dynamic requires + './preload_test.js', +]); + +const preloadConfig: Configuration = { + context, + mode: mode as Configuration['mode'], + devtool: mode === 'development' ? 'inline-source-map' : false, + entry: ['./preload.js'], + // Stack-traces have to be readable so don't mangle function names. + optimization: { + minimizer: [ + new TerserPlugin({ + terserOptions: { + mangle: { + keep_fnames: true, + }, + }, + }), + ], + }, + target: 'electron-preload', + output: { + path: resolve(context), + filename: 'preload.bundle.js', + publicPath: './', + }, + resolve: { + extensions: ['.js'], + alias: {}, + }, + externals: [ + ({ request = '' }, callback) => { + if (EXTERNAL_MODULE.has(request)) { + return callback(undefined, `commonjs2 ${request}`); + } + + callback(); + }, + ], +}; + +export default [preloadConfig]; diff --git a/webpack-sql-worker.config.ts b/webpack-sql-worker.config.ts new file mode 100644 index 000000000000..9d2ae5411ea8 --- /dev/null +++ b/webpack-sql-worker.config.ts @@ -0,0 +1,36 @@ +// Copyright 2019-2020 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { resolve, join } from 'path'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { Configuration } from 'webpack'; + +const context = __dirname; + +// A path relative to `ts/sql/` in `asar.unpacked` +const libDir = join('..', '..', 'node_modules', 'better-sqlite3'); +const bindingFile = join(libDir, 'build', 'Release', 'better_sqlite3.node'); + +const workerConfig: Configuration = { + context, + mode: 'development', + devtool: false, + entry: ['./ts/sql/mainWorker.js'], + target: 'node', + output: { + path: resolve(context, 'ts', 'sql'), + filename: 'mainWorker.bundle.js', + publicPath: './', + }, + resolve: { + extensions: ['.js'], + alias: { + bindings: join(context, 'ts', 'sql', 'mainWorkerBindings.js'), + }, + }, + externals: { + 'better_sqlite3.node': `commonjs2 ${bindingFile}`, + }, +}; + +export default [workerConfig]; diff --git a/webpack.config.ts b/webpack.config.ts index ad5a1789b4c3..9607c90489ed 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -5,7 +5,6 @@ import { resolve } from 'path'; // eslint-disable-next-line import/no-extraneous-dependencies import { Configuration, EnvironmentPlugin, ProvidePlugin } from 'webpack'; import HtmlWebpackPlugin = require('html-webpack-plugin'); -import TerserPlugin = require('terser-webpack-plugin'); const context = __dirname; const { NODE_ENV: mode = 'development' } = process.env; @@ -90,63 +89,4 @@ const stickerCreatorConfig: Configuration = { }, }; -const EXTERNAL_MODULE = new Set([ - 'backbone', - 'better-sqlite3', - 'ffi-napi', - 'fs-xattr', - 'fsevents', - 'got', - 'jquery', - 'libsignal-client', - 'node-fetch', - 'node-sass', - 'pino', - 'proxy-agent', - 'ref-array-napi', - 'ref-napi', - 'ringrtc', - 'sharp', - 'websocket', - 'zkgroup', -]); - -const preloadConfig: Configuration = { - context, - mode: mode as Configuration['mode'], - devtool: mode === 'development' ? 'inline-source-map' : false, - entry: ['./preload.js'], - // Stack-traces have to be readable so don't mangle function names. - optimization: { - minimizer: [ - new TerserPlugin({ - terserOptions: { - mangle: { - keep_fnames: true, - }, - }, - }), - ], - }, - target: 'electron-preload', - output: { - path: resolve(context), - filename: 'preload.bundle.js', - publicPath: './', - }, - resolve: { - extensions: ['.js'], - alias: {}, - }, - externals: [ - ({ request = '' }, callback) => { - if (EXTERNAL_MODULE.has(request)) { - return callback(undefined, `commonjs2 ${request}`); - } - - callback(); - }, - ], -}; - -export default [stickerCreatorConfig, preloadConfig]; +export default [stickerCreatorConfig];