Improve cold start performance

ts/sql/Interface.ts

@@ -259,7 +259,12 @@ export type ServerInterface = DataInterface & {
     configDir: string;
     key: string;
     messages: LocaleMessagesType;
-  }) => Promise<boolean>;
+  }) => Promise<void>;
+
+  initializeRenderer: (options: {
+    configDir: string;
+    key: string;
+  }) => Promise<void>;
 
   removeKnownAttachments: (
     allAttachments: Array<string>
@@ -393,7 +398,6 @@ export type ClientInterface = DataInterface & {
   // Client-side only, and test-only
 
   _removeConversations: (ids: Array<string>) => Promise<void>;
-  _jobs: { [id: string]: ClientJobType };
 };
 
 export type ClientJobType = {

ts/sql/Queueing.ts

@@ -0,0 +1,141 @@
+// Copyright 2018-2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import Queue from 'p-queue';
+import { ServerInterface } from './Interface';
+
+let allQueriesDone: () => void | undefined;
+let sqlQueries = 0;
+let singleQueue: Queue | null = null;
+let multipleQueue: Queue | null = null;
+
+// Note: we don't want queue timeouts, because delays here are due to in-progress sql
+//   operations. For example we might try to start a transaction when the prevous isn't
+//   done, causing that database operation to fail.
+function makeNewSingleQueue(): Queue {
+  singleQueue = new Queue({ concurrency: 1 });
+  return singleQueue;
+}
+function makeNewMultipleQueue(): Queue {
+  multipleQueue = new Queue({ concurrency: 10 });
+  return multipleQueue;
+}
+
+const DEBUG = false;
+
+function makeSQLJob(
+  fn: ServerInterface[keyof ServerInterface],
+  args: Array<unknown>,
+  callName: keyof ServerInterface
+) {
+  if (DEBUG) {
+    // eslint-disable-next-line no-console
+    console.log(`SQL(${callName}) queued`);
+  }
+  return async () => {
+    sqlQueries += 1;
+    const start = Date.now();
+    if (DEBUG) {
+      // eslint-disable-next-line no-console
+      console.log(`SQL(${callName}) started`);
+    }
+    let result;
+    try {
+      // Ignoring this error TS2556: Expected 3 arguments, but got 0 or more.
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      result = await fn(...args);
+    } finally {
+      sqlQueries -= 1;
+      if (allQueriesDone && sqlQueries <= 0) {
+        allQueriesDone();
+      }
+    }
+    const end = Date.now();
+    const delta = end - start;
+    if (DEBUG || delta > 10) {
+      // eslint-disable-next-line no-console
+      console.log(`SQL(${callName}) succeeded in ${end - start}ms`);
+    }
+    return result;
+  };
+}
+
+async function handleCall(
+  fn: ServerInterface[keyof ServerInterface],
+  args: Array<unknown>,
+  callName: keyof ServerInterface
+) {
+  if (!fn) {
+    throw new Error(`sql channel: ${callName} is not an available function`);
+  }
+
+  let result;
+
+  // We queue here to keep multi-query operations atomic. Without it, any multistage
+  //   data operation (even within a BEGIN/COMMIT) can become interleaved, since all
+  //   requests share one database connection.
+
+  // A needsSerial method must be run in our single concurrency queue.
+  if (fn.needsSerial) {
+    if (singleQueue) {
+      result = await singleQueue.add(makeSQLJob(fn, args, callName));
+    } else if (multipleQueue) {
+      const queue = makeNewSingleQueue();
+
+      const multipleQueueLocal = multipleQueue;
+      queue.add(() => multipleQueueLocal.onIdle());
+      multipleQueue = null;
+
+      result = await queue.add(makeSQLJob(fn, args, callName));
+    } else {
+      const queue = makeNewSingleQueue();
+      result = await queue.add(makeSQLJob(fn, args, callName));
+    }
+  } else {
+    // The request can be parallelized. To keep the same structure as the above block
+    //   we force this section into the 'lonely if' pattern.
+    // eslint-disable-next-line no-lonely-if
+    if (multipleQueue) {
+      result = await multipleQueue.add(makeSQLJob(fn, args, callName));
+    } else if (singleQueue) {
+      const queue = makeNewMultipleQueue();
+      queue.pause();
+
+      const singleQueueRef = singleQueue;
+
+      singleQueue = null;
+      const promise = queue.add(makeSQLJob(fn, args, callName));
+      if (singleQueueRef) {
+        await singleQueueRef.onIdle();
+      }
+
+      queue.start();
+      result = await promise;
+    } else {
+      const queue = makeNewMultipleQueue();
+      result = await queue.add(makeSQLJob(fn, args, callName));
+    }
+  }
+
+  return result;
+}
+
+export async function waitForPendingQueries(): Promise<void> {
+  return new Promise<void>(resolve => {
+    if (sqlQueries === 0) {
+      resolve();
+    } else {
+      allQueriesDone = () => resolve();
+    }
+  });
+}
+
+export function applyQueueing(dataInterface: ServerInterface): ServerInterface {
+  return Object.keys(dataInterface).reduce((acc, callName) => {
+    const serverInterfaceKey = callName as keyof ServerInterface;
+    acc[serverInterfaceKey] = async (...args: Array<unknown>) =>
+      handleCall(dataInterface[serverInterfaceKey], args, serverInterfaceKey);
+    return acc;
+  }, {} as ServerInterface);
+}

ts/sql/Server.ts

@@ -14,7 +14,6 @@ import mkdirp from 'mkdirp';
 import rimraf from 'rimraf';
 import PQueue from 'p-queue';
 import sql from '@journeyapps/sqlcipher';
-import { app, clipboard, dialog } from 'electron';
 
 import pify from 'pify';
 import { v4 as generateUUID } from 'uuid';
@@ -31,8 +30,6 @@ import {
   pick,
 } from 'lodash';
 
-import { redactAll } from '../../js/modules/privacy';
-import { remove as removeUserConfig } from '../../app/user_config';
 import { combineNames } from '../util/combineNames';
 
 import { GroupV2MemberType } from '../model-types.d';
@@ -54,6 +51,7 @@ import {
   StickerType,
   UnprocessedType,
 } from './Interface';
+import { applyQueueing } from './Queueing';
 
 declare global {
   // We want to extend `Function`'s properties, so we need to use an interface.
@@ -195,13 +193,14 @@ const dataInterface: ServerInterface = {
   // Server-only
 
   initialize,
+  initializeRenderer,
 
   removeKnownAttachments,
   removeKnownStickers,
   removeKnownDraftAttachments,
 };
 
-export default dataInterface;
+export default applyQueueing(dataInterface);
 
 function objectToJSON(data: any) {
   return JSON.stringify(data);
@@ -210,6 +209,14 @@ function jsonToObject(json: string): any {
   return JSON.parse(json);
 }
 
+function isRenderer() {
+  if (typeof process === 'undefined' || !process) {
+    return true;
+  }
+
+  return process.type === 'renderer';
+}
+
 async function openDatabase(filePath: string): Promise<sql.Database> {
   return new Promise((resolve, reject) => {
     let instance: sql.Database | undefined;
@@ -1702,6 +1709,7 @@ async function updateSchema(instance: PromisifiedSQLDatabase) {
 }
 
 let globalInstance: PromisifiedSQLDatabase | undefined;
+let globalInstanceRenderer: PromisifiedSQLDatabase | undefined;
 let databaseFilePath: string | undefined;
 let indexedDBPath: string | undefined;
 
@@ -1788,37 +1796,57 @@ async function initialize({
     await getMessageCount();
   } catch (error) {
     console.log('Database startup error:', error.stack);
-    const buttonIndex = dialog.showMessageBoxSync({
-      buttons: [
-        messages.copyErrorAndQuit.message,
-        messages.deleteAndRestart.message,
-      ],
-      defaultId: 0,
-      detail: redactAll(error.stack),
-      message: messages.databaseError.message,
-      noLink: true,
-      type: 'error',
-    });
-
-    if (buttonIndex === 0) {
-      clipboard.writeText(
-        `Database startup error:\n\n${redactAll(error.stack)}`
-      );
-    } else {
-      if (promisified) {
-        await promisified.close();
-      }
-      await removeDB();
-      removeUserConfig();
-      app.relaunch();
+    if (promisified) {
+      await promisified.close();
     }
+    throw error;
+  }
+}
 
-    app.exit(1);
-
-    return false;
+async function initializeRenderer({
+  configDir,
+  key,
+}: {
+  configDir: string;
+  key: string;
+}) {
+  if (!isRenderer()) {
+    throw new Error('Cannot call from main process.');
+  }
+  if (globalInstanceRenderer) {
+    throw new Error('Cannot initialize more than once!');
+  }
+  if (!isString(configDir)) {
+    throw new Error('initialize: configDir is required!');
+  }
+  if (!isString(key)) {
+    throw new Error('initialize: key is required!');
   }
 
-  return true;
+  if (!indexedDBPath) {
+    indexedDBPath = join(configDir, 'IndexedDB');
+  }
+
+  const dbDir = join(configDir, 'sql');
+
+  if (!databaseFilePath) {
+    databaseFilePath = join(dbDir, 'db.sqlite');
+  }
+
+  let promisified: PromisifiedSQLDatabase | undefined;
+
+  try {
+    promisified = await openAndSetUpSQLCipher(databaseFilePath, { key });
+
+    // At this point we can allow general access to the database
+    globalInstanceRenderer = promisified;
+
+    // test database
+    await getMessageCount();
+  } catch (error) {
+    window.log.error('Database startup error:', error.stack);
+    throw error;
+  }
 }
 
 async function close() {
@@ -1857,6 +1885,13 @@ async function removeIndexedDBFiles() {
 }
 
 function getInstance(): PromisifiedSQLDatabase {
+  if (isRenderer()) {
+    if (!globalInstanceRenderer) {
+      throw new Error('getInstance: globalInstanceRenderer not set!');
+    }
+    return globalInstanceRenderer;
+  }
+
   if (!globalInstance) {
     throw new Error('getInstance: globalInstance not set!');
   }
@@ -2285,6 +2320,7 @@ async function updateConversation(data: ConversationType) {
     }
   );
 }
+
 async function updateConversations(array: Array<ConversationType>) {
   const db = getInstance();
   await db.run('BEGIN TRANSACTION;');
@@ -2544,7 +2580,7 @@ async function saveMessage(
           `UPDATE messages SET
             id = $id,
             json = $json,
-  
+
             body = $body,
             conversationId = $conversationId,
             expirationStartTimestamp = $expirationStartTimestamp,
@@ -2616,7 +2652,7 @@ async function saveMessage(
         `INSERT INTO messages (
           id,
           json,
-  
+
           body,
           conversationId,
           expirationStartTimestamp,
@@ -2638,7 +2674,7 @@ async function saveMessage(
         ) values (
           $id,
           $json,
-  
+
           $body,
           $conversationId,
           $expirationStartTimestamp,
@@ -2967,7 +3003,7 @@ async function getLastConversationActivity({
   const row = await db.get(
     `SELECT * FROM messages WHERE
        conversationId = $conversationId AND
-       (type IS NULL 
+       (type IS NULL
         OR
         type NOT IN (
           'profile-change',

ts/sql/initialize.ts

@@ -0,0 +1,16 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { ipcRenderer as ipc } from 'electron';
+import fs from 'fs-extra';
+import pify from 'pify';
+import sql from './Server';
+
+const getRealPath = pify(fs.realpath);
+
+export async function initialize(): Promise<void> {
+  const configDir = await getRealPath(ipc.sendSync('get-user-data-path'));
+  const key = ipc.sendSync('user-config-key');
+
+  await sql.initializeRenderer({ configDir, key });
+}