Bundle sql worker with webpack

This commit is contained in:
Fedor Indutny 2021-04-14 12:43:11 -07:00 committed by Scott Nonnenberg
parent 31a777a130
commit 1ca121aef5
9 changed files with 177 additions and 110 deletions

1
.gitignore vendored
View file

@ -40,3 +40,4 @@ sticker-creator/dist/*
/.idea
/storybook-static/
preload.bundle.*
ts/sql/mainWorker.bundle.js.LICENSE.txt

View file

@ -52,7 +52,10 @@
"build:dev": "run-s --print-label build:grunt build:typed-scss build:webpack",
"build:grunt": "yarn grunt",
"build:typed-scss": "tsm sticker-creator",
"build:webpack": "cross-env NODE_ENV=production webpack",
"build:webpack": "run-p build:webpack:sticker-creator build:webpack:preload build:webpack:sql-worker",
"build:webpack:sticker-creator": "cross-env NODE_ENV=production webpack",
"build:webpack:preload": "cross-env NODE_ENV=production webpack -c webpack-preload.config.ts",
"build:webpack:sql-worker": "cross-env NODE_ENV=production webpack -c webpack-sql-worker.config.ts",
"build:electron": "electron-builder --config.extraMetadata.environment=$SIGNAL_ENV",
"build:release": "cross-env SIGNAL_ENV=production yarn build:electron -- --config.directories.output=release",
"build:zip": "node scripts/zip-macos-release.js",
@ -366,18 +369,8 @@
]
},
"asarUnpack": [
"js/modules/privacy.js",
"ts/environment.js",
"ts/logging/log.js",
"ts/logging/shared.js",
"ts/sql/Server.js",
"ts/sql/mainWorker.js",
"ts/util/assert.js",
"ts/util/combineNames.js",
"ts/util/enum.js",
"ts/util/isNormalNumber.js",
"ts/util/missingCaseError.js",
"ts/util/reallyJsonStringify.js"
"ts/sql/mainWorker.bundle.js",
"node_modules/better-sqlite3/build/Release/better_sqlite3.node"
],
"files": [
"package.json",
@ -438,7 +431,7 @@
"!node_modules/sharp/{install,src,vendor/include,vendor/*/include}",
"!node_modules/better-sqlite3/deps/*",
"!node_modules/better-sqlite3/src/*",
"node_modules/better-sqlite3/build/Release/*.node",
"node_modules/better-sqlite3/build/Release/better_sqlite3.node",
"node_modules/libsignal-client/build/*${platform}*.node",
"node_modules/ringrtc/build/${platform}/**",
"!**/node_modules/ffi-napi/deps",

View file

@ -688,36 +688,7 @@ try {
});
if (config.environment === 'test') {
// This is a hack to let us run TypeScript tests in the renderer process. See the
// code in `test/index.html`.
const pendingDescribeCalls = [];
window.describe = (...args) => {
pendingDescribeCalls.push(args);
};
/* eslint-disable global-require, import/no-extraneous-dependencies */
const fastGlob = require('fast-glob');
fastGlob
.sync('./ts/test-{both,electron}/**/*_test.js', {
absolute: true,
cwd: __dirname,
})
.forEach(require);
delete window.describe;
window.test = {
pendingDescribeCalls,
fastGlob,
normalizePath: require('normalize-path'),
fse: require('fs-extra'),
tmp: require('tmp'),
path: require('path'),
basePath: __dirname,
attachmentsPath: window.Signal.Migrations.attachmentsPath,
};
/* eslint-enable global-require, import/no-extraneous-dependencies */
require('./preload_test.js');
}
} catch (error) {
/* eslint-disable no-console */

31
preload_test.js Normal file
View file

@ -0,0 +1,31 @@
/* global window */
// This is a hack to let us run TypeScript tests in the renderer process. See the
// code in `test/index.html`.
const pendingDescribeCalls = [];
window.describe = (...args) => {
pendingDescribeCalls.push(args);
};
/* eslint-disable global-require, import/no-extraneous-dependencies */
const fastGlob = require('fast-glob');
fastGlob
.sync('./ts/test-{both,electron}/**/*_test.js', {
absolute: true,
cwd: __dirname,
})
.forEach(require);
delete window.describe;
window.test = {
pendingDescribeCalls,
fastGlob,
normalizePath: require('normalize-path'),
fse: require('fs-extra'),
tmp: require('tmp'),
path: require('path'),
basePath: __dirname,
attachmentsPath: window.Signal.Migrations.attachmentsPath,
};

View file

@ -4,6 +4,8 @@
import { join } from 'path';
import { Worker } from 'worker_threads';
const ASAR_PATTERN = /app\.asar$/;
export type InitializeOptions = {
readonly configDir: string;
readonly key: string;
@ -56,12 +58,18 @@ export class MainSQL {
private onResponse = new Map<number, PromisePair<any>>();
constructor() {
const appDir = join(__dirname, '..', '..').replace(
/app\.asar$/,
'app.asar.unpacked'
);
let appDir = join(__dirname, '..', '..');
let isBundled = false;
this.worker = new Worker(join(appDir, 'ts', 'sql', 'mainWorker.js'));
if (ASAR_PATTERN.test(appDir)) {
appDir = appDir.replace(ASAR_PATTERN, 'app.asar.unpacked');
isBundled = true;
}
const scriptDir = join(appDir, 'ts', 'sql');
this.worker = new Worker(
join(scriptDir, isBundled ? 'mainWorker.bundle.js' : 'mainWorker.js')
);
this.worker.on('message', (wrappedResponse: WrappedWorkerResponse) => {
const { seq, error, response } = wrappedResponse;

View file

@ -0,0 +1,13 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
// This is a shim that gets inserted in place of `bindings` npm module when
// building sql worker bundle.
module.exports = (binding: string) => {
if (binding === 'better_sqlite3.node') {
// eslint-disable-next-line global-require, import/no-unresolved
return require('better_sqlite3.node');
}
throw new Error(`Unknown binding ${binding}`);
};

74
webpack-preload.config.ts Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2019-2020 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { resolve } from 'path';
// eslint-disable-next-line import/no-extraneous-dependencies
import { Configuration } from 'webpack';
import TerserPlugin = require('terser-webpack-plugin');
const context = __dirname;
const { NODE_ENV: mode = 'development' } = process.env;
const EXTERNAL_MODULE = new Set([
'backbone',
'better-sqlite3',
'ffi-napi',
'fs-xattr',
'fsevents',
'got',
'jquery',
'libsignal-client',
'node-fetch',
'node-sass',
'pino',
'proxy-agent',
'ref-array-napi',
'ref-napi',
'ringrtc',
'sharp',
'websocket',
'zkgroup',
// Uses fast-glob and dynamic requires
'./preload_test.js',
]);
const preloadConfig: Configuration = {
context,
mode: mode as Configuration['mode'],
devtool: mode === 'development' ? 'inline-source-map' : false,
entry: ['./preload.js'],
// Stack-traces have to be readable so don't mangle function names.
optimization: {
minimizer: [
new TerserPlugin({
terserOptions: {
mangle: {
keep_fnames: true,
},
},
}),
],
},
target: 'electron-preload',
output: {
path: resolve(context),
filename: 'preload.bundle.js',
publicPath: './',
},
resolve: {
extensions: ['.js'],
alias: {},
},
externals: [
({ request = '' }, callback) => {
if (EXTERNAL_MODULE.has(request)) {
return callback(undefined, `commonjs2 ${request}`);
}
callback();
},
],
};
export default [preloadConfig];

View file

@ -0,0 +1,36 @@
// Copyright 2019-2020 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { resolve, join } from 'path';
// eslint-disable-next-line import/no-extraneous-dependencies
import { Configuration } from 'webpack';
const context = __dirname;
// A path relative to `ts/sql/` in `asar.unpacked`
const libDir = join('..', '..', 'node_modules', 'better-sqlite3');
const bindingFile = join(libDir, 'build', 'Release', 'better_sqlite3.node');
const workerConfig: Configuration = {
context,
mode: 'development',
devtool: false,
entry: ['./ts/sql/mainWorker.js'],
target: 'node',
output: {
path: resolve(context, 'ts', 'sql'),
filename: 'mainWorker.bundle.js',
publicPath: './',
},
resolve: {
extensions: ['.js'],
alias: {
bindings: join(context, 'ts', 'sql', 'mainWorkerBindings.js'),
},
},
externals: {
'better_sqlite3.node': `commonjs2 ${bindingFile}`,
},
};
export default [workerConfig];

View file

@ -5,7 +5,6 @@ import { resolve } from 'path';
// eslint-disable-next-line import/no-extraneous-dependencies
import { Configuration, EnvironmentPlugin, ProvidePlugin } from 'webpack';
import HtmlWebpackPlugin = require('html-webpack-plugin');
import TerserPlugin = require('terser-webpack-plugin');
const context = __dirname;
const { NODE_ENV: mode = 'development' } = process.env;
@ -90,63 +89,4 @@ const stickerCreatorConfig: Configuration = {
},
};
const EXTERNAL_MODULE = new Set([
'backbone',
'better-sqlite3',
'ffi-napi',
'fs-xattr',
'fsevents',
'got',
'jquery',
'libsignal-client',
'node-fetch',
'node-sass',
'pino',
'proxy-agent',
'ref-array-napi',
'ref-napi',
'ringrtc',
'sharp',
'websocket',
'zkgroup',
]);
const preloadConfig: Configuration = {
context,
mode: mode as Configuration['mode'],
devtool: mode === 'development' ? 'inline-source-map' : false,
entry: ['./preload.js'],
// Stack-traces have to be readable so don't mangle function names.
optimization: {
minimizer: [
new TerserPlugin({
terserOptions: {
mangle: {
keep_fnames: true,
},
},
}),
],
},
target: 'electron-preload',
output: {
path: resolve(context),
filename: 'preload.bundle.js',
publicPath: './',
},
resolve: {
extensions: ['.js'],
alias: {},
},
externals: [
({ request = '' }, callback) => {
if (EXTERNAL_MODULE.has(request)) {
return callback(undefined, `commonjs2 ${request}`);
}
callback();
},
],
};
export default [stickerCreatorConfig, preloadConfig];
export default [stickerCreatorConfig];