Bundle sql worker with webpack

.gitignore

@@ -40,3 +40,4 @@ sticker-creator/dist/*
 /.idea
 /storybook-static/
 preload.bundle.*
+ts/sql/mainWorker.bundle.js.LICENSE.txt

package.json

@@ -52,7 +52,10 @@
     "build:dev": "run-s --print-label build:grunt build:typed-scss build:webpack",
     "build:grunt": "yarn grunt",
     "build:typed-scss": "tsm sticker-creator",
-    "build:webpack": "cross-env NODE_ENV=production webpack",
+    "build:webpack": "run-p build:webpack:sticker-creator build:webpack:preload build:webpack:sql-worker",
+    "build:webpack:sticker-creator": "cross-env NODE_ENV=production webpack",
+    "build:webpack:preload": "cross-env NODE_ENV=production webpack -c webpack-preload.config.ts",
+    "build:webpack:sql-worker": "cross-env NODE_ENV=production webpack -c webpack-sql-worker.config.ts",
     "build:electron": "electron-builder --config.extraMetadata.environment=$SIGNAL_ENV",
     "build:release": "cross-env SIGNAL_ENV=production yarn build:electron -- --config.directories.output=release",
     "build:zip": "node scripts/zip-macos-release.js",
@@ -366,18 +369,8 @@
       ]
     },
     "asarUnpack": [
-      "js/modules/privacy.js",
+      "ts/sql/mainWorker.bundle.js",
-      "ts/environment.js",
+      "node_modules/better-sqlite3/build/Release/better_sqlite3.node"
-      "ts/logging/log.js",
-      "ts/logging/shared.js",
-      "ts/sql/Server.js",
-      "ts/sql/mainWorker.js",
-      "ts/util/assert.js",
-      "ts/util/combineNames.js",
-      "ts/util/enum.js",
-      "ts/util/isNormalNumber.js",
-      "ts/util/missingCaseError.js",
-      "ts/util/reallyJsonStringify.js"
     ],
     "files": [
       "package.json",
@@ -438,7 +431,7 @@
       "!node_modules/sharp/{install,src,vendor/include,vendor/*/include}",
       "!node_modules/better-sqlite3/deps/*",
       "!node_modules/better-sqlite3/src/*",
-      "node_modules/better-sqlite3/build/Release/*.node",
+      "node_modules/better-sqlite3/build/Release/better_sqlite3.node",
       "node_modules/libsignal-client/build/*${platform}*.node",
       "node_modules/ringrtc/build/${platform}/**",
       "!**/node_modules/ffi-napi/deps",

preload.js

@@ -688,36 +688,7 @@ try {
   });
 
   if (config.environment === 'test') {
-    // This is a hack to let us run TypeScript tests in the renderer process. See the
+    require('./preload_test.js');
-    //   code in `test/index.html`.
-    const pendingDescribeCalls = [];
-    window.describe = (...args) => {
-      pendingDescribeCalls.push(args);
-    };
-
-    /* eslint-disable global-require, import/no-extraneous-dependencies */
-    const fastGlob = require('fast-glob');
-
-    fastGlob
-      .sync('./ts/test-{both,electron}/**/*_test.js', {
-        absolute: true,
-        cwd: __dirname,
-      })
-      .forEach(require);
-
-    delete window.describe;
-
-    window.test = {
-      pendingDescribeCalls,
-      fastGlob,
-      normalizePath: require('normalize-path'),
-      fse: require('fs-extra'),
-      tmp: require('tmp'),
-      path: require('path'),
-      basePath: __dirname,
-      attachmentsPath: window.Signal.Migrations.attachmentsPath,
-    };
-    /* eslint-enable global-require, import/no-extraneous-dependencies */
   }
 } catch (error) {
   /* eslint-disable no-console */

preload_test.js

@@ -0,0 +1,31 @@
+/* global window */
+
+// This is a hack to let us run TypeScript tests in the renderer process. See the
+//   code in `test/index.html`.
+const pendingDescribeCalls = [];
+window.describe = (...args) => {
+  pendingDescribeCalls.push(args);
+};
+
+/* eslint-disable global-require, import/no-extraneous-dependencies */
+const fastGlob = require('fast-glob');
+
+fastGlob
+  .sync('./ts/test-{both,electron}/**/*_test.js', {
+    absolute: true,
+    cwd: __dirname,
+  })
+  .forEach(require);
+
+delete window.describe;
+
+window.test = {
+  pendingDescribeCalls,
+  fastGlob,
+  normalizePath: require('normalize-path'),
+  fse: require('fs-extra'),
+  tmp: require('tmp'),
+  path: require('path'),
+  basePath: __dirname,
+  attachmentsPath: window.Signal.Migrations.attachmentsPath,
+};

ts/sql/main.ts

@@ -4,6 +4,8 @@
 import { join } from 'path';
 import { Worker } from 'worker_threads';
 
+const ASAR_PATTERN = /app\.asar$/;
+
 export type InitializeOptions = {
   readonly configDir: string;
   readonly key: string;
@@ -56,12 +58,18 @@ export class MainSQL {
   private onResponse = new Map<number, PromisePair<any>>();
 
   constructor() {
-    const appDir = join(__dirname, '..', '..').replace(
+    let appDir = join(__dirname, '..', '..');
-      /app\.asar$/,
+    let isBundled = false;
-      'app.asar.unpacked'
-    );
 
-    this.worker = new Worker(join(appDir, 'ts', 'sql', 'mainWorker.js'));
+    if (ASAR_PATTERN.test(appDir)) {
+      appDir = appDir.replace(ASAR_PATTERN, 'app.asar.unpacked');
+      isBundled = true;
+    }
+
+    const scriptDir = join(appDir, 'ts', 'sql');
+    this.worker = new Worker(
+      join(scriptDir, isBundled ? 'mainWorker.bundle.js' : 'mainWorker.js')
+    );
 
     this.worker.on('message', (wrappedResponse: WrappedWorkerResponse) => {
       const { seq, error, response } = wrappedResponse;

ts/sql/mainWorkerBindings.ts

@@ -0,0 +1,13 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+// This is a shim that gets inserted in place of `bindings` npm module when
+// building sql worker bundle.
+module.exports = (binding: string) => {
+  if (binding === 'better_sqlite3.node') {
+    // eslint-disable-next-line global-require, import/no-unresolved
+    return require('better_sqlite3.node');
+  }
+
+  throw new Error(`Unknown binding ${binding}`);
+};

webpack-preload.config.ts

@@ -0,0 +1,74 @@
+// Copyright 2019-2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { resolve } from 'path';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { Configuration } from 'webpack';
+import TerserPlugin = require('terser-webpack-plugin');
+
+const context = __dirname;
+const { NODE_ENV: mode = 'development' } = process.env;
+
+const EXTERNAL_MODULE = new Set([
+  'backbone',
+  'better-sqlite3',
+  'ffi-napi',
+  'fs-xattr',
+  'fsevents',
+  'got',
+  'jquery',
+  'libsignal-client',
+  'node-fetch',
+  'node-sass',
+  'pino',
+  'proxy-agent',
+  'ref-array-napi',
+  'ref-napi',
+  'ringrtc',
+  'sharp',
+  'websocket',
+  'zkgroup',
+
+  // Uses fast-glob and dynamic requires
+  './preload_test.js',
+]);
+
+const preloadConfig: Configuration = {
+  context,
+  mode: mode as Configuration['mode'],
+  devtool: mode === 'development' ? 'inline-source-map' : false,
+  entry: ['./preload.js'],
+  // Stack-traces have to be readable so don't mangle function names.
+  optimization: {
+    minimizer: [
+      new TerserPlugin({
+        terserOptions: {
+          mangle: {
+            keep_fnames: true,
+          },
+        },
+      }),
+    ],
+  },
+  target: 'electron-preload',
+  output: {
+    path: resolve(context),
+    filename: 'preload.bundle.js',
+    publicPath: './',
+  },
+  resolve: {
+    extensions: ['.js'],
+    alias: {},
+  },
+  externals: [
+    ({ request = '' }, callback) => {
+      if (EXTERNAL_MODULE.has(request)) {
+        return callback(undefined, `commonjs2 ${request}`);
+      }
+
+      callback();
+    },
+  ],
+};
+
+export default [preloadConfig];

webpack-sql-worker.config.ts

@@ -0,0 +1,36 @@
+// Copyright 2019-2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { resolve, join } from 'path';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { Configuration } from 'webpack';
+
+const context = __dirname;
+
+// A path relative to `ts/sql/` in `asar.unpacked`
+const libDir = join('..', '..', 'node_modules', 'better-sqlite3');
+const bindingFile = join(libDir, 'build', 'Release', 'better_sqlite3.node');
+
+const workerConfig: Configuration = {
+  context,
+  mode: 'development',
+  devtool: false,
+  entry: ['./ts/sql/mainWorker.js'],
+  target: 'node',
+  output: {
+    path: resolve(context, 'ts', 'sql'),
+    filename: 'mainWorker.bundle.js',
+    publicPath: './',
+  },
+  resolve: {
+    extensions: ['.js'],
+    alias: {
+      bindings: join(context, 'ts', 'sql', 'mainWorkerBindings.js'),
+    },
+  },
+  externals: {
+    'better_sqlite3.node': `commonjs2 ${bindingFile}`,
+  },
+};
+
+export default [workerConfig];

webpack.config.ts

@@ -5,7 +5,6 @@ import { resolve } from 'path';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { Configuration, EnvironmentPlugin, ProvidePlugin } from 'webpack';
 import HtmlWebpackPlugin = require('html-webpack-plugin');
-import TerserPlugin = require('terser-webpack-plugin');
 
 const context = __dirname;
 const { NODE_ENV: mode = 'development' } = process.env;
@@ -90,63 +89,4 @@ const stickerCreatorConfig: Configuration = {
   },
 };
 
-const EXTERNAL_MODULE = new Set([
+export default [stickerCreatorConfig];
-  'backbone',
-  'better-sqlite3',
-  'ffi-napi',
-  'fs-xattr',
-  'fsevents',
-  'got',
-  'jquery',
-  'libsignal-client',
-  'node-fetch',
-  'node-sass',
-  'pino',
-  'proxy-agent',
-  'ref-array-napi',
-  'ref-napi',
-  'ringrtc',
-  'sharp',
-  'websocket',
-  'zkgroup',
-]);
-
-const preloadConfig: Configuration = {
-  context,
-  mode: mode as Configuration['mode'],
-  devtool: mode === 'development' ? 'inline-source-map' : false,
-  entry: ['./preload.js'],
-  // Stack-traces have to be readable so don't mangle function names.
-  optimization: {
-    minimizer: [
-      new TerserPlugin({
-        terserOptions: {
-          mangle: {
-            keep_fnames: true,
-          },
-        },
-      }),
-    ],
-  },
-  target: 'electron-preload',
-  output: {
-    path: resolve(context),
-    filename: 'preload.bundle.js',
-    publicPath: './',
-  },
-  resolve: {
-    extensions: ['.js'],
-    alias: {},
-  },
-  externals: [
-    ({ request = '' }, callback) => {
-      if (EXTERNAL_MODULE.has(request)) {
-        return callback(undefined, `commonjs2 ${request}`);
-      }
-
-      callback();
-    },
-  ],
-};
-
-export default [stickerCreatorConfig, preloadConfig];