Moves SQL to full IPC

app/main.ts

@@ -1529,7 +1529,6 @@ const onDatabaseError = async (error: string) => {
   ready = false;
 
   if (mainWindow) {
-    drop(settingsChannel?.invokeCallbackInMainWindow('closeDB', []));
     mainWindow.close();
   }
   mainWindow = undefined;

test/test.js

@@ -23,17 +23,6 @@ before(async () => {
   window.testUtilities.installMessageController();
 
   await deleteIndexedDB();
-  try {
-    window.SignalContext.log.info('Initializing SQL in renderer');
-    const isTesting = true;
-    await window.Signal.Data.startInRenderer(isTesting);
-    window.SignalContext.log.info('SQL initialized in renderer');
-  } catch (err) {
-    window.SignalContext.log.error(
-      'SQL failed to initialize',
-      err && err.stack ? err.stack : err
-    );
-  }
   await window.testUtilities.initializeMessageCounter();
   await window.Signal.Data.removeAll();
   await window.storage.fetch();

ts/background.ts

@@ -944,12 +944,6 @@ export async function startApp(): Promise<void> {
       void window.Signal.Data.ensureFilePermissions();
     }
 
-    try {
-      await window.Signal.Data.startInRendererProcess();
-    } catch (err) {
-      log.error('SQL failed to initialize', Errors.toLogFormat(err));
-    }
-
     setAppLoadingScreenMessage(window.i18n('icu:loading'), window.i18n);
 
     let isMigrationWithIndexComplete = false;
@@ -2590,9 +2584,6 @@ export async function startApp(): Promise<void> {
     // Start listeners here, after we get through our queue.
     RotateSignedPreKeyListener.init(window.Whisper.events, newVersion);
 
-    // Go back to main process before processing delayed actions
-    await window.Signal.Data.goBackToMainProcess();
-
     profileKeyResponseQueue.start();
     lightSessionResetQueue.start();
     onDecryptionErrorQueue.start();

ts/services/networkObserver.ts

@@ -8,7 +8,6 @@ import type {
 import { getSocketStatus } from '../shims/socketStatus';
 import * as log from '../logging/log';
 import { SECOND } from '../util/durations';
-import { SocketStatus } from '../types/SocketStatus';
 
 type NetworkActions = {
   checkNetworkStatus: (x: CheckNetworkStatusPayloadType) => NetworkActionType;
@@ -23,12 +22,6 @@ export function initializeNetworkObserver(
   const refresh = () => {
     const socketStatus = getSocketStatus();
 
-    if (socketStatus === SocketStatus.CLOSED) {
-      // If we couldn't connect during startup - we should still switch SQL to
-      // the main process to avoid stalling UI.
-      void window.Signal.Data.goBackToMainProcess();
-    }
-
     networkActions.checkNetworkStatus({
       isOnline: navigator.onLine,
       socketStatus,

ts/sql/Client.ts

@@ -2,29 +2,16 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { ipcRenderer as ipc } from 'electron';
-import fs from 'fs-extra';
-import pify from 'pify';
 import PQueue from 'p-queue';
 
-import {
+import { has, get, groupBy, isTypedArray, last, map, omit } from 'lodash';
-  compact,
-  fromPairs,
-  groupBy,
-  isFunction,
-  isTypedArray,
-  last,
-  map,
-  omit,
-  toPairs,
-} from 'lodash';
 
 import { deleteExternalFiles } from '../types/Conversation';
 import { expiringMessagesDeletionService } from '../services/expiringMessagesDeletion';
 import { tapToViewMessagesDeletionService } from '../services/tapToViewMessagesDeletionService';
 import * as Bytes from '../Bytes';
 import { createBatcher } from '../util/batcher';
-import { explodePromise } from '../util/explodePromise';
+import { assertDev, softAssert } from '../util/assert';
-import { assertDev, softAssert, strictAssert } from '../util/assert';
 import { mapObjectWithSpec } from '../util/mapObjectWithSpec';
 import type { ObjectMappingSpecType } from '../util/mapObjectWithSpec';
 import { cleanDataForIpc } from './cleanDataForIpc';
@@ -38,6 +25,7 @@ import type { StoredJob } from '../jobs/types';
 import { formatJobForInsert } from '../jobs/formatJobForInsert';
 import { cleanupMessage } from '../util/cleanup';
 import { drop } from '../util/drop';
+import { ipcInvoke, doShutdown } from './channels';
 
 import type {
   AdjacentMessagesByConversationOptionsType,
@@ -65,18 +53,11 @@ import type {
   SignedPreKeyType,
   StoredSignedPreKeyType,
 } from './Interface';
-import Server from './Server';
-import { parseSqliteError, SqliteErrorKind } from './errors';
 import { MINUTE } from '../util/durations';
 import { getMessageIdForLogging } from '../util/idForLogging';
 import type { MessageAttributesType } from '../model-types';
 import { incrementMessageCounter } from '../util/incrementMessageCounter';
 
-const getRealPath = pify(fs.realpath);
-
-const MIN_TRACE_DURATION = 10;
-
-const SQL_CHANNEL_KEY = 'sql-channel';
 const ERASE_SQL_KEY = 'erase-sql-key';
 const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
 const ERASE_STICKERS_KEY = 'erase-stickers';
@@ -85,92 +66,6 @@ const ERASE_DRAFTS_KEY = 'erase-drafts';
 const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments';
 const ENSURE_FILE_PERMISSIONS = 'ensure-file-permissions';
 
-enum RendererState {
-  InMain = 'InMain',
-  Opening = 'Opening',
-  InRenderer = 'InRenderer',
-  Closing = 'Closing',
-}
-
-let activeJobCount = 0;
-let resolveShutdown: (() => void) | undefined;
-let shutdownPromise: Promise<void> | null = null;
-
-let state = RendererState.InMain;
-const startupQueries = new Map<string, number>();
-
-async function startInRendererProcess(isTesting = false): Promise<void> {
-  strictAssert(
-    state === RendererState.InMain,
-    `startInRendererProcess: expected ${state} to be ${RendererState.InMain}`
-  );
-
-  log.info('data.startInRendererProcess: switching to renderer process');
-  state = RendererState.Opening;
-
-  if (!isTesting) {
-    await ipc.invoke('database-ready');
-  }
-
-  const configDir = await getRealPath(ipc.sendSync('get-user-data-path'));
-  const key = ipc.sendSync('user-config-key');
-
-  await Server.initializeRenderer({ configDir, key });
-
-  log.info('data.startInRendererProcess: switched to renderer process');
-
-  state = RendererState.InRenderer;
-}
-
-async function goBackToMainProcess(): Promise<void> {
-  if (state === RendererState.InMain) {
-    log.info('goBackToMainProcess: Already in the main process');
-    return;
-  }
-
-  strictAssert(
-    state === RendererState.InRenderer,
-    `goBackToMainProcess: expected ${state} to be ${RendererState.InRenderer}`
-  );
-
-  // We don't need to wait for pending queries since they are synchronous.
-  log.info('data.goBackToMainProcess: switching to main process');
-  const closePromise = channels.close();
-
-  // It should be the last query we run in renderer process
-  state = RendererState.Closing;
-  await closePromise;
-  state = RendererState.InMain;
-
-  // Print query statistics for whole startup
-  const entries = Array.from(startupQueries.entries());
-  startupQueries.clear();
-
-  // Sort by decreasing duration
-  entries
-    .sort((a, b) => b[1] - a[1])
-    .filter(([_, duration]) => duration > MIN_TRACE_DURATION)
-    .forEach(([query, duration]) => {
-      log.info(`startup query: ${query} ${duration}ms`);
-    });
-
-  log.info('data.goBackToMainProcess: switched to main process');
-}
-
-const channelsAsUnknown = fromPairs(
-  compact(
-    map(toPairs(Server), ([name, value]: [string, unknown]) => {
-      if (isFunction(value)) {
-        return [name, makeChannel(name)];
-      }
-
-      return null;
-    })
-  )
-) as unknown;
-
-const channels: ServerInterface = channelsAsUnknown as ServerInterface;
-
 const exclusiveInterface: ClientExclusiveInterface = {
   createOrUpdateIdentityKey,
   getIdentityKeyById,
@@ -209,30 +104,53 @@ const exclusiveInterface: ClientExclusiveInterface = {
   removeOtherData,
   cleanupOrphanedAttachments,
   ensureFilePermissions,
-
-  // Client-side only, and test-only
-
-  startInRendererProcess,
-  goBackToMainProcess,
 };
 
-// Because we can't force this module to conform to an interface, we narrow our exports
+type ClientOverridesType = ClientExclusiveInterface &
-//   to this one default export, which does conform to the interface.
+  Pick<
-// Note: In Javascript, you need to access the .default property when requiring it
+    ServerInterface,
-// https://github.com/microsoft/TypeScript/issues/420
+    | 'removeMessage'
-const dataInterface: ClientInterface = {
+    | 'removeMessages'
-  ...channels,
+    | 'saveAttachmentDownloadJob'
-  ...exclusiveInterface,
+    | 'saveMessage'
+    | 'saveMessages'
+    | 'updateConversations'
+  >;
 
-  // Overrides
+const channels: ServerInterface = new Proxy({} as ServerInterface, {
-  updateConversations,
+  get(_target, name) {
-  saveMessage,
+    return async (...args: ReadonlyArray<unknown>) =>
-  saveMessages,
+      ipcInvoke(String(name), args);
+  },
+});
+
+const clientExclusiveOverrides: ClientOverridesType = {
+  ...exclusiveInterface,
   removeMessage,
   removeMessages,
   saveAttachmentDownloadJob,
+  saveMessage,
+  saveMessages,
+  updateConversations,
 };
 
+const dataInterface: ClientInterface = new Proxy(
+  {
+    ...clientExclusiveOverrides,
+  } as ClientInterface,
+  {
+    get(target, name) {
+      return async (...args: ReadonlyArray<unknown>) => {
+        if (has(target, name)) {
+          return get(target, name)(...args);
+        }
+
+        return get(channels, name)(...args);
+      };
+    },
+  }
+);
+
 export default dataInterface;
 
 function _cleanData(
@@ -272,99 +190,6 @@ export function _cleanMessageData(data: MessageType): MessageType {
   return _cleanData(omit(result, ['dataMessage']));
 }
 
-async function doShutdown() {
-  log.info(
-    `data.shutdown: shutdown requested. ${activeJobCount} jobs outstanding`
-  );
-
-  if (shutdownPromise) {
-    return shutdownPromise;
-  }
-
-  // No outstanding jobs, return immediately
-  if (activeJobCount === 0) {
-    return;
-  }
-
-  ({ promise: shutdownPromise, resolve: resolveShutdown } =
-    explodePromise<void>());
-
-  try {
-    await shutdownPromise;
-  } finally {
-    log.info('data.shutdown: process complete');
-  }
-}
-
-function makeChannel(fnName: string) {
-  return async (...args: ReadonlyArray<unknown>) => {
-    // During startup we want to avoid the high overhead of IPC so we utilize
-    // the db that exists in the renderer process to be able to boot up quickly
-    // once the app is running we switch back to the main process to avoid the
-    // UI from locking up whenever we do costly db operations.
-    if (state === RendererState.InRenderer) {
-      const serverFnName = fnName as keyof ServerInterface;
-      const serverFn = Server[serverFnName] as (
-        ...fnArgs: ReadonlyArray<unknown>
-      ) => unknown;
-      const start = Date.now();
-
-      try {
-        // Ignoring this error TS2556: Expected 3 arguments, but got 0 or more.
-        return await serverFn(...args);
-      } catch (error) {
-        const sqliteErrorKind = parseSqliteError(error);
-        if (sqliteErrorKind === SqliteErrorKind.Corrupted) {
-          log.error(
-            'Detected sql corruption in renderer process. ' +
-              `Restarting the application immediately. Error: ${error.message}`
-          );
-          ipc?.send('database-error', error.stack);
-        } else if (sqliteErrorKind === SqliteErrorKind.Readonly) {
-          log.error(`Detected readonly sql database: ${error.message}`);
-          ipc?.send('database-readonly');
-        }
-
-        log.error(
-          `Renderer SQL channel job (${fnName}) error ${error.message}`
-        );
-        throw error;
-      } finally {
-        const duration = Date.now() - start;
-
-        startupQueries.set(
-          serverFnName,
-          (startupQueries.get(serverFnName) || 0) + duration
-        );
-
-        if (duration > MIN_TRACE_DURATION) {
-          log.info(
-            `Renderer SQL channel job (${fnName}) completed in ${duration}ms`
-          );
-        }
-      }
-    }
-
-    if (shutdownPromise && fnName !== 'close') {
-      throw new Error(
-        `Rejecting SQL channel job (${fnName}); application is shutting down`
-      );
-    }
-
-    activeJobCount += 1;
-    return createTaskWithTimeout(async () => {
-      try {
-        return await ipc.invoke(SQL_CHANNEL_KEY, fnName, ...args);
-      } finally {
-        activeJobCount -= 1;
-        if (activeJobCount === 0) {
-          resolveShutdown?.();
-        }
-      }
-    }, `SQL channel call (${fnName})`)();
-  };
-}
-
 function specToBytes<Input, Output>(
   spec: ObjectMappingSpecType,
   data: Input

ts/sql/Interface.ts

@@ -830,10 +830,6 @@ export type ServerInterface = DataInterface & {
     key: string;
     logger: LoggerType;
   }) => Promise<void>;
-  initializeRenderer: (options: {
-    configDir: string;
-    key: string;
-  }) => Promise<void>;
 
   getKnownMessageAttachments: (
     cursor?: MessageAttachmentsCursorType
@@ -913,11 +909,6 @@ export type ClientExclusiveInterface = {
   removeOtherData: () => Promise<void>;
   cleanupOrphanedAttachments: () => Promise<void>;
   ensureFilePermissions: () => Promise<void>;
-
-  // To decide whether to use IPC to use the database in the main process or
-  //   use the db already running in the renderer.
-  goBackToMainProcess: () => Promise<void>;
-  startInRendererProcess: (isTesting?: boolean) => Promise<void>;
 };
 
 export type ClientInterface = DataInterface & ClientExclusiveInterface;

ts/sql/Server.ts

@@ -365,7 +365,6 @@ const dataInterface: ServerInterface = {
   // Server-only
 
   initialize,
-  initializeRenderer,
 
   getKnownMessageAttachments,
   finishGetKnownMessageAttachments,
@@ -431,14 +430,6 @@ function rowToSticker(row: StickerRow): StickerType {
   };
 }
 
-function isRenderer() {
-  if (typeof process === 'undefined' || !process) {
-    return true;
-  }
-
-  return process.type === 'renderer';
-}
-
 function keyDatabase(db: Database, key: string): void {
   // https://www.zetetic.net/sqlcipher/sqlcipher-api/#key
   db.pragma(`key = "x'${key}'"`);
@@ -522,7 +513,6 @@ function openAndSetUpSQLCipher(filePath: string, { key }: { key: string }) {
 
 let globalInstance: Database | undefined;
 let logger = consoleLogger;
-let globalInstanceRenderer: Database | undefined;
 let databaseFilePath: string | undefined;
 let indexedDBPath: string | undefined;
 
@@ -583,63 +573,13 @@ async function initialize({
   }
 }
 
-async function initializeRenderer({
-  configDir,
-  key,
-}: {
-  configDir: string;
-  key: string;
-}): Promise<void> {
-  if (!isRenderer()) {
-    throw new Error('Cannot call from main process.');
-  }
-  if (globalInstanceRenderer) {
-    throw new Error('Cannot initialize more than once!');
-  }
-  if (!isString(configDir)) {
-    throw new Error('initialize: configDir is required!');
-  }
-  if (!isString(key)) {
-    throw new Error('initialize: key is required!');
-  }
-
-  if (!indexedDBPath) {
-    indexedDBPath = join(configDir, 'IndexedDB');
-  }
-
-  const dbDir = join(configDir, 'sql');
-
-  if (!databaseFilePath) {
-    databaseFilePath = join(dbDir, 'db.sqlite');
-  }
-
-  let promisified: Database | undefined;
-
-  try {
-    promisified = openAndSetUpSQLCipher(databaseFilePath, { key });
-
-    // At this point we can allow general access to the database
-    globalInstanceRenderer = promisified;
-
-    // test database
-    getMessageCountSync();
-  } catch (error) {
-    log.error('Database startup error:', error.stack);
-    throw error;
-  }
-}
-
 async function close(): Promise<void> {
-  for (const dbRef of [globalInstanceRenderer, globalInstance]) {
+  // SQLLite documentation suggests that we run `PRAGMA optimize` right
-    // SQLLite documentation suggests that we run `PRAGMA optimize` right
+  // before closing the database connection.
-    // before closing the database connection.
+  globalInstance?.pragma('optimize');
-    dbRef?.pragma('optimize');
-
-    dbRef?.close();
-  }
 
+  globalInstance?.close();
   globalInstance = undefined;
-  globalInstanceRenderer = undefined;
 }
 
 async function removeDB(): Promise<void> {
@@ -676,13 +616,6 @@ async function removeIndexedDBFiles(): Promise<void> {
 }
 
 function getInstance(): Database {
-  if (isRenderer()) {
-    if (!globalInstanceRenderer) {
-      throw new Error('getInstance: globalInstanceRenderer not set!');
-    }
-    return globalInstanceRenderer;
-  }
-
   if (!globalInstance) {
     throw new Error('getInstance: globalInstance not set!');
   }

ts/sql/channels.ts

@@ -0,0 +1,61 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { ipcRenderer } from 'electron';
+import * as log from '../logging/log';
+import createTaskWithTimeout from '../textsecure/TaskWithTimeout';
+import { explodePromise } from '../util/explodePromise';
+
+const SQL_CHANNEL_KEY = 'sql-channel';
+let activeJobCount = 0;
+let resolveShutdown: (() => void) | undefined;
+let shutdownPromise: Promise<void> | null = null;
+
+export async function ipcInvoke(
+  name: string,
+  args: ReadonlyArray<unknown>
+): Promise<void> {
+  const fnName = String(name);
+
+  if (shutdownPromise && name !== 'close') {
+    throw new Error(
+      `Rejecting SQL channel job (${fnName}); application is shutting down`
+    );
+  }
+
+  activeJobCount += 1;
+  return createTaskWithTimeout(async () => {
+    try {
+      return await ipcRenderer.invoke(SQL_CHANNEL_KEY, name, ...args);
+    } finally {
+      activeJobCount -= 1;
+      if (activeJobCount === 0) {
+        resolveShutdown?.();
+      }
+    }
+  }, `SQL channel call (${fnName})`)();
+}
+
+export async function doShutdown(): Promise<void> {
+  log.info(
+    `data.shutdown: shutdown requested. ${activeJobCount} jobs outstanding`
+  );
+
+  if (shutdownPromise) {
+    return shutdownPromise;
+  }
+
+  // No outstanding jobs, return immediately
+  if (activeJobCount === 0) {
+    return;
+  }
+
+  ({ promise: shutdownPromise, resolve: resolveShutdown } =
+    explodePromise<void>());
+
+  try {
+    await shutdownPromise;
+  } finally {
+    log.info('data.shutdown: process complete');
+  }
+}

ts/util/createIPCEvents.ts

@@ -102,7 +102,6 @@ export type IPCEventsCallbacksType = {
   authorizeArtCreator: (data: AuthorizeArtCreatorDataType) => void;
   deleteAllData: () => Promise<void>;
   deleteAllMyStories: () => Promise<void>;
-  closeDB: () => Promise<void>;
   editCustomColor: (colorId: string, customColor: CustomColorType) => void;
   getConversationsWithCustomColor: (x: string) => Array<ConversationType>;
   installStickerPack: (packId: string, key: string) => Promise<void>;
@@ -477,15 +476,9 @@ export function createIPCEvents(
       window.reduxActions.globalModals.showShortcutGuideModal(),
 
     deleteAllData: async () => {
-      await window.Signal.Data.goBackToMainProcess();
-
       renderClearingDataView();
     },
 
-    closeDB: async () => {
-      await window.Signal.Data.goBackToMainProcess();
-    },
-
     showStickerPack: (packId, key) => {
       // We can get these events even if the user has never linked this instance.
       if (!Registration.everDone()) {

ts/windows/main/start.ts

@@ -20,6 +20,7 @@ import { start as startConversationController } from '../../ConversationControll
 import { MessageController } from '../../util/MessageController';
 import { Environment, getEnvironment } from '../../environment';
 import { isProduction } from '../../util/version';
+import { ipcInvoke } from '../../sql/channels';
 
 window.addEventListener('contextmenu', e => {
   const node = e.target as Element | null;
@@ -46,7 +47,6 @@ startConversationController();
 
 if (!isProduction(window.SignalContext.getVersion())) {
   const SignalDebug = {
-    Data: window.Signal.Data,
     cdsLookup: (options: CdsLookupOptionsType) =>
       window.textsecure.server?.cdsLookup(options),
     getConversation: (id: string) => window.ConversationController.get(id),
@@ -67,6 +67,8 @@ if (!isProduction(window.SignalContext.getVersion())) {
     setSfuUrl: (url: string) => {
       window.Signal.Services.calling._sfuUrl = url;
     },
+    sqlCall: (name: string, ...args: ReadonlyArray<unknown>) =>
+      ipcInvoke(name, args),
   };
 
   contextBridge.exposeInMainWorld('SignalDebug', SignalDebug);

ts/windows/preload.ts

@@ -15,7 +15,6 @@ installCallback('resetDefaultChatColor');
 installCallback('setGlobalDefaultConversationColor');
 installCallback('getDefaultConversationColor');
 installCallback('persistZoomFactor');
-installCallback('closeDB');
 
 // Getters only. These are set by the primary device
 installSetting('blockedCount', {