Improve cold start performance

about_preload.js

@@ -25,4 +25,4 @@ window.closeAbout = () => ipcRenderer.send('close-about');
 
 window.i18n = i18n.setup(locale, localeMessages);
 
-require('./ts/logging/set_up_renderer_logging');
+require('./ts/logging/set_up_renderer_logging').initialize();

debug_log_preload.js

@@ -32,7 +32,7 @@ window.getEnvironment = getEnvironment;
 window.Backbone = require('backbone');
 require('./ts/backbone/views/whisper_view');
 require('./ts/backbone/views/toast_view');
-require('./ts/logging/set_up_renderer_logging');
+require('./ts/logging/set_up_renderer_logging').initialize();
 
 window.closeDebugLog = () => ipcRenderer.send('close-debug-log');
 window.Backbone = require('backbone');

js/expiring_messages.js

@@ -18,18 +18,25 @@
         MessageCollection: Whisper.MessageCollection,
       });
 
-      await Promise.all(
+      const messageIds = [];
-        messages.map(async fromDB => {
+      const inMemoryMessages = [];
-          const message = MessageController.register(fromDB.id, fromDB);
+      const messageCleanup = [];
 
-          window.log.info('Message expired', {
+      messages.forEach(dbMessage => {
-            sentAt: message.get('sent_at'),
+        const message = MessageController.register(dbMessage.id, dbMessage);
+        messageIds.push(message.id);
+        inMemoryMessages.push(message);
+        messageCleanup.push(message.cleanup());
       });
 
       // We delete after the trigger to allow the conversation time to process
       //   the expiration before the message is removed from the database.
-          await window.Signal.Data.removeMessage(message.id, {
+      await window.Signal.Data.removeMessages(messageIds);
-            Message: Whisper.Message,
+      await Promise.all(messageCleanup);
+
+      inMemoryMessages.forEach(message => {
+        window.log.info('Message expired', {
+          sentAt: message.get('sent_at'),
         });
 
         Whisper.events.trigger(
@@ -42,8 +49,7 @@
         if (conversation) {
           conversation.trigger('expired', message);
         }
-        })
+      });
-      );
     } catch (error) {
       window.log.error(
         'destroyExpiredMessages: Error deleting expired messages',

js/expiring_tap_to_view_messages.js

@@ -58,7 +58,9 @@
       return;
     }
 
-    const nextCheck = toAgeOut.get('received_at') + THIRTY_DAYS;
+    const receivedAt =
+      toAgeOut.get('received_at_ms') || toAgeOut.get('received_at');
+    const nextCheck = receivedAt + THIRTY_DAYS;
 
     Whisper.TapToViewMessagesListener.nextCheck = nextCheck;
     window.log.info(

js/message_controller.js

@@ -6,6 +6,8 @@
   window.Whisper = window.Whisper || {};
 
   const messageLookup = Object.create(null);
+  const msgIDsBySender = new Map();
+  const msgIDsBySentAt = new Map();
 
   const SECOND = 1000;
   const MINUTE = SECOND * 60;
@@ -31,10 +33,18 @@
       timestamp: Date.now(),
     };
 
+    msgIDsBySentAt.set(message.get('sent_at'), id);
+    msgIDsBySender.set(message.getSenderIdentifier(), id);
+
     return message;
   }
 
   function unregister(id) {
+    const { message } = messageLookup[id] || {};
+    if (message) {
+      msgIDsBySender.delete(message.getSenderIdentifier());
+      msgIDsBySentAt.delete(message.get('sent_at'));
+    }
     delete messageLookup[id];
   }
 
@@ -50,7 +60,7 @@
         now - timestamp > FIVE_MINUTES &&
         (!conversation || !conversation.messageCollection.length)
       ) {
-        delete messageLookup[message.id];
+        unregister(message.id);
       }
     }
   }
@@ -60,6 +70,22 @@
     return existing && existing.message ? existing.message : null;
   }
 
+  function findBySentAt(sentAt) {
+    const id = msgIDsBySentAt.get(sentAt);
+    if (!id) {
+      return null;
+    }
+    return getById(id);
+  }
+
+  function findBySender(sender) {
+    const id = msgIDsBySender.get(sender);
+    if (!id) {
+      return null;
+    }
+    return getById(id);
+  }
+
   function _get() {
     return messageLookup;
   }
@@ -70,6 +96,8 @@
     register,
     unregister,
     cleanup,
+    findBySender,
+    findBySentAt,
     getById,
     _get,
   };

libtextsecure/test/_test.js

@@ -67,3 +67,16 @@ window.Whisper.events = {
   on() {},
   trigger() {},
 };
+
+before(async () => {
+  try {
+    window.log.info('Initializing SQL in renderer');
+    await window.sqlInitializer.initialize();
+    window.log.info('SQL initialized in renderer');
+  } catch (err) {
+    window.log.error(
+      'SQL failed to initialize',
+      err && err.stack ? err.stack : err
+    );
+  }
+});

main.js

@@ -19,6 +19,8 @@ const electron = require('electron');
 const packageJson = require('./package.json');
 const GlobalErrors = require('./app/global_errors');
 const { setup: setupSpellChecker } = require('./app/spell_check');
+const { redactAll } = require('./js/modules/privacy');
+const removeUserConfig = require('./app/user_config').remove;
 
 GlobalErrors.addHandler();
 
@@ -30,6 +32,7 @@ const getRealPath = pify(fs.realpath);
 const {
   app,
   BrowserWindow,
+  clipboard,
   dialog,
   ipcMain: ipc,
   Menu,
@@ -1058,12 +1061,37 @@ app.on('ready', async () => {
     loadingWindow.loadURL(prepareURL([__dirname, 'loading.html']));
   });
 
-  const success = await sqlInitPromise;
+  try {
-
+    await sqlInitPromise;
-  if (!success) {
+  } catch (error) {
     console.log('sql.initialize was unsuccessful; returning early');
+    const buttonIndex = dialog.showMessageBoxSync({
+      buttons: [
+        locale.messages.copyErrorAndQuit.message,
+        locale.messages.deleteAndRestart.message,
+      ],
+      defaultId: 0,
+      detail: redactAll(error.stack),
+      message: locale.messages.databaseError.message,
+      noLink: true,
+      type: 'error',
+    });
+
+    if (buttonIndex === 0) {
+      clipboard.writeText(
+        `Database startup error:\n\n${redactAll(error.stack)}`
+      );
+    } else {
+      await sql.removeDB();
+      removeUserConfig();
+      app.relaunch();
+    }
+
+    app.exit(1);
+
     return;
   }
+
   // eslint-disable-next-line more/no-then
   appStartInitialSpellcheckSetting = await getSpellCheckSetting();
   await sqlChannels.initialize();
@@ -1075,10 +1103,10 @@ app.on('ready', async () => {
       await sql.removeIndexedDBFiles();
       await sql.removeItemById(IDB_KEY);
     }
-  } catch (error) {
+  } catch (err) {
     console.log(
       '(ready event handler) error deleting IndexedDB:',
-      error && error.stack ? error.stack : error
+      err && err.stack ? err.stack : err
     );
   }
 
@@ -1113,10 +1141,10 @@ app.on('ready', async () => {
 
   try {
     await attachments.clearTempPath(userDataPath);
-  } catch (error) {
+  } catch (err) {
     logger.error(
       'main/ready: Error deleting temp dir:',
-      error && error.stack ? error.stack : error
+      err && err.stack ? err.stack : err
     );
   }
   await attachmentChannel.initialize({
@@ -1458,6 +1486,17 @@ ipc.on('locale-data', event => {
   event.returnValue = locale.messages;
 });
 
+// Used once to initialize SQL in the renderer process
+ipc.once('user-config-key', event => {
+  // eslint-disable-next-line no-param-reassign
+  event.returnValue = userConfig.get('key');
+});
+
+ipc.on('get-user-data-path', event => {
+  // eslint-disable-next-line no-param-reassign
+  event.returnValue = app.getPath('userData');
+});
+
 function getDataFromMainWindow(name, callback) {
   ipc.once(`get-success-${name}`, (_event, error, value) =>
     callback(error, value)

permissions_popup_preload.js

@@ -49,7 +49,7 @@ window.subscribeToSystemThemeChange = fn => {
   });
 };
 
-require('./ts/logging/set_up_renderer_logging');
+require('./ts/logging/set_up_renderer_logging').initialize();
 
 window.closePermissionsPopup = () =>
   ipcRenderer.send('close-permissions-popup');

preload.js

@@ -22,6 +22,8 @@ try {
   const { app } = remote;
   const { nativeTheme } = remote.require('electron');
 
+  window.sqlInitializer = require('./ts/sql/initialize');
+
   window.PROTO_ROOT = 'protos';
   const config = require('url').parse(window.location.toString(), true).query;
 
@@ -400,7 +402,7 @@ try {
 
   // We pull these dependencies in now, from here, because they have Node.js dependencies
 
-  require('./ts/logging/set_up_renderer_logging');
+  require('./ts/logging/set_up_renderer_logging').initialize();
 
   if (config.proxyUrl) {
     window.log.info('Using provided proxy url');

settings_preload.js

@@ -129,4 +129,4 @@ function makeSetter(name) {
 window.Backbone = require('backbone');
 require('./ts/backbone/views/whisper_view');
 require('./ts/backbone/views/toast_view');
-require('./ts/logging/set_up_renderer_logging');
+require('./ts/logging/set_up_renderer_logging').initialize();

test/_test.js

@@ -76,6 +76,16 @@ function deleteIndexedDB() {
 /* Delete the database before running any tests */
 before(async () => {
   await deleteIndexedDB();
+  try {
+    window.log.info('Initializing SQL in renderer');
+    await window.sqlInitializer.initialize();
+    window.log.info('SQL initialized in renderer');
+  } catch (err) {
+    window.log.error(
+      'SQL failed to initialize',
+      err && err.stack ? err.stack : err
+    );
+  }
   await window.Signal.Data.removeAll();
   await window.storage.fetch();
 });

ts/background.ts

@@ -3,8 +3,19 @@
 
 import { DataMessageClass } from './textsecure.d';
 import { WhatIsThis } from './window.d';
+import { assert } from './util/assert';
 
 export async function startApp(): Promise<void> {
+  try {
+    window.log.info('Initializing SQL in renderer');
+    await window.sqlInitializer.initialize();
+    window.log.info('SQL initialized in renderer');
+  } catch (err) {
+    window.log.error(
+      'SQL failed to initialize',
+      err && err.stack ? err.stack : err
+    );
+  }
   const eventHandlerQueue = new window.PQueue({
     concurrency: 1,
     timeout: 1000 * 60 * 2,
@@ -1615,6 +1626,9 @@ export async function startApp(): Promise<void> {
   let connectCount = 0;
   let connecting = false;
   async function connect(firstRun?: boolean) {
+    window.receivedAtCounter =
+      window.storage.get('lastReceivedAtCounter') || Date.now();
+
     if (connecting) {
       window.log.warn('connect already running', { connectCount });
       return;
@@ -2038,7 +2052,7 @@ export async function startApp(): Promise<void> {
       logger: window.log,
     });
 
-    let interval: NodeJS.Timer | null = setInterval(() => {
+    let interval: NodeJS.Timer | null = setInterval(async () => {
       const view = window.owsDesktopApp.appView;
       if (view) {
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -2046,6 +2060,24 @@ export async function startApp(): Promise<void> {
         interval = null;
         view.onEmpty();
         window.logAppLoadedEvent();
+        const attachmentDownloadQueue = window.attachmentDownloadQueue || [];
+        const THREE_DAYS_AGO = Date.now() - 3600 * 72 * 1000;
+        const MAX_ATTACHMENT_MSGS_TO_DOWNLOAD = 250;
+        const attachmentsToDownload = attachmentDownloadQueue.filter(
+          (message, index) =>
+            index <= MAX_ATTACHMENT_MSGS_TO_DOWNLOAD ||
+            message.getReceivedAt() < THREE_DAYS_AGO
+        );
+        window.log.info(
+          'Downloading recent attachments of total attachments',
+          attachmentsToDownload.length,
+          attachmentDownloadQueue.length
+        );
+        await Promise.all(
+          attachmentsToDownload.map(message =>
+            message.queueAttachmentDownloads()
+          )
+        );
       }
     }, 500);
 
@@ -2646,14 +2678,15 @@ export async function startApp(): Promise<void> {
       sent_at: data.timestamp,
       serverTimestamp: data.serverTimestamp,
       sent_to: sentTo,
-      received_at: now,
+      received_at: data.receivedAtCounter,
+      received_at_ms: data.receivedAtDate,
       conversationId: descriptor.id,
       type: 'outgoing',
       sent: true,
       unidentifiedDeliveries: data.unidentifiedDeliveries || [],
       expirationStartTimestamp: Math.min(
-        data.expirationStartTimestamp || data.timestamp || Date.now(),
+        data.expirationStartTimestamp || data.timestamp || now,
-        Date.now()
+        now
       ),
     } as WhatIsThis);
   }
@@ -2856,13 +2889,18 @@ export async function startApp(): Promise<void> {
     data: WhatIsThis,
     descriptor: MessageDescriptor
   ) {
+    assert(
+      Boolean(data.receivedAtCounter),
+      `Did not receive receivedAtCounter for message: ${data.timestamp}`
+    );
     return new window.Whisper.Message({
       source: data.source,
       sourceUuid: data.sourceUuid,
       sourceDevice: data.sourceDevice,
       sent_at: data.timestamp,
       serverTimestamp: data.serverTimestamp,
-      received_at: Date.now(),
+      received_at: data.receivedAtCounter,
+      received_at_ms: data.receivedAtDate,
       conversationId: descriptor.id,
       unidentifiedDeliveryReceived: data.unidentifiedDeliveryReceived,
       type: 'incoming',

ts/components/LightboxGallery.stories.tsx

@@ -40,7 +40,8 @@ story.add('Image and Video', () => {
         message: {
           attachments: [],
           id: 'image-msg',
-          received_at: Date.now(),
+          received_at: 1,
+          received_at_ms: Date.now(),
         },
         objectURL: '/fixtures/tina-rolf-269345-unsplash.jpg',
       },
@@ -55,7 +56,8 @@ story.add('Image and Video', () => {
         message: {
           attachments: [],
           id: 'video-msg',
-          received_at: Date.now(),
+          received_at: 2,
+          received_at_ms: Date.now(),
         },
         objectURL: '/fixtures/pixabay-Soap-Bubble-7141.mp4',
       },
@@ -79,7 +81,8 @@ story.add('Missing Media', () => {
         message: {
           attachments: [],
           id: 'image-msg',
-          received_at: Date.now(),
+          received_at: 3,
+          received_at_ms: Date.now(),
         },
         objectURL: undefined,
       },

ts/components/conversation/media-gallery/AttachmentSection.stories.tsx

@@ -52,7 +52,8 @@ const createRandomFile = (
     contentType,
     message: {
       id: random(now).toString(),
-      received_at: random(startTime, startTime + timeWindow),
+      received_at: Math.floor(Math.random() * 10),
+      received_at_ms: random(startTime, startTime + timeWindow),
       attachments: [],
     },
     attachment: {

ts/components/conversation/media-gallery/AttachmentSection.tsx

@@ -9,6 +9,7 @@ import { MediaGridItem } from './MediaGridItem';
 import { MediaItemType } from '../../LightboxGallery';
 import { missingCaseError } from '../../../util/missingCaseError';
 import { LocalizerType } from '../../../types/Util';
+import { getMessageTimestamp } from '../../../util/getMessageTimestamp';
 
 export type Props = {
   i18n: LocalizerType;
@@ -58,7 +59,7 @@ export class AttachmentSection extends React.Component<Props> {
               fileSize={attachment.size}
               shouldShowSeparator={shouldShowSeparator}
               onClick={onClick}
-              timestamp={message.received_at}
+              timestamp={getMessageTimestamp(message)}
             />
           );
         default:

ts/components/conversation/media-gallery/MediaGallery.tsx

@@ -12,6 +12,7 @@ import { groupMediaItemsByDate } from './groupMediaItemsByDate';
 import { ItemClickEvent } from './types/ItemClickEvent';
 import { missingCaseError } from '../../../util/missingCaseError';
 import { LocalizerType } from '../../../types/Util';
+import { getMessageTimestamp } from '../../../util/getMessageTimestamp';
 
 import { MediaItemType } from '../../LightboxGallery';
 
@@ -145,7 +146,7 @@ export class MediaGallery extends React.Component<Props, State> {
     const sections = groupMediaItemsByDate(now, mediaItems).map(section => {
       const first = section.mediaItems[0];
       const { message } = first;
-      const date = moment(message.received_at);
+      const date = moment(getMessageTimestamp(message));
       const header =
         section.type === 'yearMonth'
           ? date.format(MONTH_FORMAT)

ts/components/conversation/media-gallery/groupMediaItemsByDate.ts

@@ -5,6 +5,7 @@ import moment from 'moment';
 import { compact, groupBy, sortBy } from 'lodash';
 
 import { MediaItemType } from '../../LightboxGallery';
+import { getMessageTimestamp } from '../../../util/getMessageTimestamp';
 
 // import { missingCaseError } from '../../../util/missingCaseError';
 
@@ -120,7 +121,7 @@ const withSection = (referenceDateTime: moment.Moment) => (
   const thisMonth = moment(referenceDateTime).startOf('month');
 
   const { message } = mediaItem;
-  const mediaItemReceivedDate = moment.utc(message.received_at);
+  const mediaItemReceivedDate = moment.utc(getMessageTimestamp(message));
   if (mediaItemReceivedDate.isAfter(today)) {
     return {
       order: 0,

ts/components/conversation/media-gallery/types/Message.ts

@@ -9,4 +9,6 @@ export type Message = {
   // Assuming this is for the API
   // eslint-disable-next-line camelcase
   received_at: number;
+  // eslint-disable-next-line camelcase
+  received_at_ms: number;
 };

ts/groups.ts

@@ -2452,7 +2452,8 @@ async function updateGroup({
     return {
       ...changeMessage,
       conversationId: conversation.id,
-      received_at: finalReceivedAt,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: finalReceivedAt,
       sent_at: syntheticSentAt,
     };
   });

ts/logging/main_process_logging.ts

@@ -61,9 +61,9 @@ export async function initialize(): Promise<bunyan> {
     }, 500);
   }
 
-  const logFile = path.join(logPath, 'log.log');
+  const logFile = path.join(logPath, 'main.log');
   const loggerOptions: bunyan.LoggerOptions = {
-    name: 'log',
+    name: 'main',
     streams: [
       {
         type: 'rotating-file',
@@ -83,31 +83,6 @@ export async function initialize(): Promise<bunyan> {
 
   const logger = bunyan.createLogger(loggerOptions);
 
-  ipc.on('batch-log', (_first, batch: unknown) => {
-    if (!Array.isArray(batch)) {
-      logger.error(
-        'batch-log IPC event was called with a non-array; dropping logs'
-      );
-      return;
-    }
-
-    batch.forEach(item => {
-      if (isLogEntry(item)) {
-        const levelString = getLogLevelString(item.level);
-        logger[levelString](
-          {
-            time: item.time,
-          },
-          item.msg
-        );
-      } else {
-        logger.error(
-          'batch-log IPC event was called with an invalid log entry; dropping entry'
-        );
-      }
-    });
-  });
-
   ipc.on('fetch-log', event => {
     fetch(logPath).then(
       data => {

ts/logging/set_up_renderer_logging.ts

@@ -7,11 +7,11 @@
 
 import { ipcRenderer as ipc } from 'electron';
 import _ from 'lodash';
-import { levelFromName } from 'bunyan';
+import * as path from 'path';
+import * as bunyan from 'bunyan';
 
 import { uploadDebugLogs } from './debuglogs';
 import { redactAll } from '../../js/modules/privacy';
-import { createBatcher } from '../util/batcher';
 import {
   LogEntryType,
   LogLevel,
@@ -23,7 +23,7 @@ import * as log from './log';
 import { reallyJsonStringify } from '../util/reallyJsonStringify';
 
 // To make it easier to visually scan logs, we make all levels the same length
-const levelMaxLength: number = Object.keys(levelFromName).reduce(
+const levelMaxLength: number = Object.keys(bunyan.levelFromName).reduce(
   (maxLength, level) => Math.max(maxLength, level.length),
   0
 );
@@ -96,6 +96,30 @@ function fetch(): Promise<string> {
   });
 }
 
+let globalLogger: undefined | bunyan;
+
+export function initialize(): void {
+  if (globalLogger) {
+    throw new Error('Already called initialize!');
+  }
+
+  const basePath = ipc.sendSync('get-user-data-path');
+  const logFile = path.join(basePath, 'logs', 'app.log');
+  const loggerOptions: bunyan.LoggerOptions = {
+    name: 'app',
+    streams: [
+      {
+        type: 'rotating-file',
+        path: logFile,
+        period: '1d',
+        count: 3,
+      },
+    ],
+  };
+
+  globalLogger = bunyan.createLogger(loggerOptions);
+}
+
 const publish = uploadDebugLogs;
 
 // A modern logging interface for the browser
@@ -103,14 +127,6 @@ const publish = uploadDebugLogs;
 const env = window.getEnvironment();
 const IS_PRODUCTION = env === 'production';
 
-const ipcBatcher = createBatcher({
-  wait: 500,
-  maxSize: 500,
-  processBatch: (items: Array<LogEntryType>) => {
-    ipc.send('batch-log', items);
-  },
-});
-
 // The Bunyan API: https://github.com/trentm/node-bunyan#log-method-api
 function logAtLevel(level: LogLevel, ...args: ReadonlyArray<unknown>): void {
   if (!IS_PRODUCTION) {
@@ -120,11 +136,16 @@ function logAtLevel(level: LogLevel, ...args: ReadonlyArray<unknown>): void {
     console._log(prefix, now(), ...args);
   }
 
-  ipcBatcher.add({
+  const levelString = getLogLevelString(level);
-    level,
+  const msg = cleanArgs(args);
-    msg: cleanArgs(args),
+  const time = new Date().toISOString();
-    time: new Date().toISOString(),
+
-  });
+  if (!globalLogger) {
+    throw new Error('Logger has not been initialized yet');
+    return;
+  }
+
+  globalLogger[levelString]({ time }, msg);
 }
 
 log.setLogAtLevel(logAtLevel);

ts/model-types.d.ts

@@ -134,6 +134,7 @@ export type MessageAttributesType = {
   groupV2Change?: GroupV2ChangeType;
   // Required. Used to sort messages in the database for the conversation timeline.
   received_at?: number;
+  received_at_ms?: number;
   // More of a legacy feature, needed as we were updating the schema of messages in the
   //   background, when we were still in IndexedDB, before attachments had gone to disk
   // We set this so that the idle message upgrade process doesn't pick this message up

ts/models/conversations.ts

@@ -2282,7 +2282,8 @@ export class ConversationModel extends window.Backbone.Model<
       conversationId: this.id,
       type: 'chat-session-refreshed',
       sent_at: receivedAt,
-      received_at: receivedAt,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: receivedAt,
       unread: 1,
       // TODO: DESKTOP-722
       // this type does not fully implement the interface it is expected to
@@ -2315,7 +2316,8 @@ export class ConversationModel extends window.Backbone.Model<
       conversationId: this.id,
       type: 'keychange',
       sent_at: this.get('timestamp'),
-      received_at: timestamp,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: timestamp,
       key_changed: keyChangedId,
       unread: 1,
       // TODO: DESKTOP-722
@@ -2373,7 +2375,8 @@ export class ConversationModel extends window.Backbone.Model<
       conversationId: this.id,
       type: 'verified-change',
       sent_at: lastMessage,
-      received_at: timestamp,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: timestamp,
       verifiedChanged: verifiedChangeId,
       verified,
       local: options.local,
@@ -2435,7 +2438,8 @@ export class ConversationModel extends window.Backbone.Model<
       conversationId: this.id,
       type: 'call-history',
       sent_at: timestamp,
-      received_at: timestamp,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: timestamp,
       unread,
       callHistoryDetails: detailsToSave,
       // TODO: DESKTOP-722
@@ -2481,11 +2485,13 @@ export class ConversationModel extends window.Backbone.Model<
     profileChange: unknown,
     conversationId?: string
   ): Promise<void> {
+    const now = Date.now();
     const message = ({
       conversationId: this.id,
       type: 'profile-change',
-      sent_at: Date.now(),
+      sent_at: now,
-      received_at: Date.now(),
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: now,
       unread: true,
       changedId: conversationId || this.id,
       profileChange,
@@ -2984,7 +2990,8 @@ export class ConversationModel extends window.Backbone.Model<
         type: 'outgoing',
         conversationId: this.get('id'),
         sent_at: timestamp,
-        received_at: timestamp,
+        received_at: window.Signal.Util.incrementMessageCounter(),
+        received_at_ms: timestamp,
         recipients,
         deletedForEveryoneTimestamp: targetTimestamp,
         // TODO: DESKTOP-722
@@ -3093,7 +3100,8 @@ export class ConversationModel extends window.Backbone.Model<
         type: 'outgoing',
         conversationId: this.get('id'),
         sent_at: timestamp,
-        received_at: timestamp,
+        received_at: window.Signal.Util.incrementMessageCounter(),
+        received_at_ms: timestamp,
         recipients,
         reaction: outgoingReaction,
         // TODO: DESKTOP-722
@@ -3244,7 +3252,8 @@ export class ConversationModel extends window.Backbone.Model<
         preview,
         attachments,
         sent_at: now,
-        received_at: now,
+        received_at: window.Signal.Util.incrementMessageCounter(),
+        received_at_ms: now,
         expireTimer,
         recipients,
         sticker,
@@ -3866,7 +3875,8 @@ export class ConversationModel extends window.Backbone.Model<
       conversationId: this.id,
       // No type; 'incoming' messages are specially treated by conversation.markRead()
       sent_at: timestamp,
-      received_at: timestamp,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: timestamp,
       flags:
         window.textsecure.protobuf.DataMessage.Flags.EXPIRATION_TIMER_UPDATE,
       expirationTimerUpdate: {
@@ -3970,7 +3980,8 @@ export class ConversationModel extends window.Backbone.Model<
       conversationId: this.id,
       // No type; 'incoming' messages are specially treated by conversation.markRead()
       sent_at: timestamp,
-      received_at: timestamp,
+      received_at: window.Signal.Util.incrementMessageCounter(),
+      received_at_ms: timestamp,
       // TODO: DESKTOP-722
     } as unknown) as MessageAttributesType);
 
@@ -4003,7 +4014,8 @@ export class ConversationModel extends window.Backbone.Model<
         conversationId: this.id,
         type: 'outgoing',
         sent_at: now,
-        received_at: now,
+        received_at: window.Signal.Util.incrementMessageCounter(),
+        received_at_ms: now,
         destination: this.get('e164'),
         destinationUuid: this.get('uuid'),
         recipients: this.getRecipients(),
@@ -4059,7 +4071,8 @@ export class ConversationModel extends window.Backbone.Model<
         conversationId: this.id,
         type: 'outgoing',
         sent_at: now,
-        received_at: now,
+        received_at: window.Signal.Util.incrementMessageCounter(),
+        received_at_ms: now,
         // TODO: DESKTOP-722
       } as unknown) as MessageAttributesType);
 

ts/models/messages.ts

@@ -198,6 +198,29 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     };
   }
 
+  getSenderIdentifier(): string {
+    const sentAt = this.get('sent_at');
+    const source = this.get('source');
+    const sourceUuid = this.get('sourceUuid');
+    const sourceDevice = this.get('sourceDevice');
+
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const sourceId = window.ConversationController.ensureContactIds({
+      e164: source,
+      uuid: sourceUuid,
+    })!;
+
+    return `${sourceId}.${sourceDevice}-${sentAt}`;
+  }
+
+  getReceivedAt(): number {
+    // We would like to get the received_at_ms ideally since received_at is
+    // now an incrementing counter for messages and not the actual time that
+    // the message was received. If this field doesn't exist on the message
+    // then we can trust received_at.
+    return Number(this.get('received_at_ms') || this.get('received_at'));
+  }
+
   isNormalBubble(): boolean {
     return (
       !this.isCallHistory() &&
@@ -381,7 +404,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
 
     return {
       sentAt: this.get('sent_at'),
-      receivedAt: this.get('received_at'),
+      receivedAt: this.getReceivedAt(),
       message: {
         ...this.getPropsForMessage(),
         disableMenu: true,
@@ -1904,9 +1927,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     window.Whisper.Notifications.removeBy({ messageId: this.id });
 
     if (!skipSave) {
-      await window.Signal.Data.saveMessage(this.attributes, {
+      window.Signal.Util.updateMessageBatcher.add(this.attributes);
-        Message: window.Whisper.Message,
-      });
     }
   }
 
@@ -1955,9 +1976,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
 
       const id = this.get('id');
       if (id && !skipSave) {
-        await window.Signal.Data.saveMessage(this.attributes, {
+        window.Signal.Util.updateMessageBatcher.add(this.attributes);
-          Message: window.Whisper.Message,
-        });
       }
     }
   }
@@ -2822,6 +2841,14 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
       }
     );
 
+    const inMemoryMessage = window.MessageController.findBySentAt(id);
+
+    let queryMessage;
+
+    if (inMemoryMessage) {
+      queryMessage = inMemoryMessage;
+    } else {
+      window.log.info('copyFromQuotedMessage: db lookup needed', id);
       const collection = await window.Signal.Data.getMessagesBySentAt(id, {
         MessageCollection: window.Whisper.MessageCollection,
       });
@@ -2835,7 +2862,11 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
         quote.referencedMessageNotFound = true;
         return message;
       }
-    if (found.isTapToView()) {
+
+      queryMessage = window.MessageController.register(found.id, found);
+    }
+
+    if (queryMessage.isTapToView()) {
       quote.text = null;
       quote.attachments = [
         {
@@ -2846,7 +2877,6 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
       return message;
     }
 
-    const queryMessage = window.MessageController.register(found.id, found);
     quote.text = queryMessage.get('body');
     if (firstAttachment) {
       firstAttachment.thumbnail = null;
@@ -2946,9 +2976,20 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
       );
 
       // First, check for duplicates. If we find one, stop processing here.
-      const existingMessage = await getMessageBySender(this.attributes, {
+      const inMemoryMessage = window.MessageController.findBySender(
+        this.getSenderIdentifier()
+      );
+      if (!inMemoryMessage) {
+        window.log.info(
+          'handleDataMessage: duplicate check db lookup needed',
+          this.getSenderIdentifier()
+        );
+      }
+      const existingMessage =
+        inMemoryMessage ||
+        (await getMessageBySender(this.attributes, {
           Message: window.Whisper.Message,
-      });
+        }));
       const isUpdate = Boolean(data && data.isRecipientUpdate);
 
       if (existingMessage && type === 'incoming') {
@@ -3422,7 +3463,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
                     dataMessage.expireTimer,
                     source,
                     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                    message.get('received_at')!,
+                    message.getReceivedAt()!,
                     {
                       fromGroupUpdate: message.isGroupUpdate(),
                     }
@@ -3437,7 +3478,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
                   undefined,
                   source,
                   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                  message.get('received_at')!
+                  message.getReceivedAt()!
                 );
               }
             }
@@ -3573,7 +3614,8 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
           (this.getConversation()!.getAccepted() || message.isOutgoing()) &&
           !shouldHoldOffDownload
         ) {
-          await message.queueAttachmentDownloads();
+          window.attachmentDownloadQueue = window.attachmentDownloadQueue || [];
+          window.attachmentDownloadQueue.unshift(message);
         }
 
         // Does this message have any pending, previously-received associated reactions?
@@ -3591,24 +3633,11 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
           )
         );
 
-        await window.Signal.Data.saveMessage(message.attributes, {
+        window.log.info(
-          Message: window.Whisper.Message,
+          'handleDataMessage: Batching save for',
-          forceSave: true,
+          message.get('sent_at')
-        });
+        );
-
+        this.saveAndNotify(conversation, confirm);
-        conversation.trigger('newmessage', message);
-
-        if (message.get('unread')) {
-          await conversation.notify(message);
-        }
-
-        // Increment the sent message count if this is an outgoing message
-        if (type === 'outgoing') {
-          conversation.incrementSentMessageCount();
-        }
-
-        window.Whisper.events.trigger('incrementProgress');
-        confirm();
       } catch (error) {
         const errorForLog = error && error.stack ? error.stack : error;
         window.log.error(
@@ -3622,6 +3651,29 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     });
   }
 
+  async saveAndNotify(
+    conversation: ConversationModel,
+    confirm: () => void
+  ): Promise<void> {
+    await window.Signal.Util.saveNewMessageBatcher.add(this.attributes);
+
+    window.log.info('Message saved', this.get('sent_at'));
+
+    conversation.trigger('newmessage', this);
+
+    if (this.get('unread')) {
+      await conversation.notify(this);
+    }
+
+    // Increment the sent message count if this is an outgoing message
+    if (this.get('type') === 'outgoing') {
+      conversation.incrementSentMessageCount();
+    }
+
+    window.Whisper.events.trigger('incrementProgress');
+    confirm();
+  }
+
   async handleReaction(
     reaction: typeof window.WhatIsThis,
     shouldPersist = true

ts/sql/Interface.ts

@@ -259,7 +259,12 @@ export type ServerInterface = DataInterface & {
     configDir: string;
     key: string;
     messages: LocaleMessagesType;
-  }) => Promise<boolean>;
+  }) => Promise<void>;
+
+  initializeRenderer: (options: {
+    configDir: string;
+    key: string;
+  }) => Promise<void>;
 
   removeKnownAttachments: (
     allAttachments: Array<string>
@@ -393,7 +398,6 @@ export type ClientInterface = DataInterface & {
   // Client-side only, and test-only
 
   _removeConversations: (ids: Array<string>) => Promise<void>;
-  _jobs: { [id: string]: ClientJobType };
 };
 
 export type ClientJobType = {

ts/sql/Queueing.ts

@@ -0,0 +1,141 @@
+// Copyright 2018-2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import Queue from 'p-queue';
+import { ServerInterface } from './Interface';
+
+let allQueriesDone: () => void | undefined;
+let sqlQueries = 0;
+let singleQueue: Queue | null = null;
+let multipleQueue: Queue | null = null;
+
+// Note: we don't want queue timeouts, because delays here are due to in-progress sql
+//   operations. For example we might try to start a transaction when the prevous isn't
+//   done, causing that database operation to fail.
+function makeNewSingleQueue(): Queue {
+  singleQueue = new Queue({ concurrency: 1 });
+  return singleQueue;
+}
+function makeNewMultipleQueue(): Queue {
+  multipleQueue = new Queue({ concurrency: 10 });
+  return multipleQueue;
+}
+
+const DEBUG = false;
+
+function makeSQLJob(
+  fn: ServerInterface[keyof ServerInterface],
+  args: Array<unknown>,
+  callName: keyof ServerInterface
+) {
+  if (DEBUG) {
+    // eslint-disable-next-line no-console
+    console.log(`SQL(${callName}) queued`);
+  }
+  return async () => {
+    sqlQueries += 1;
+    const start = Date.now();
+    if (DEBUG) {
+      // eslint-disable-next-line no-console
+      console.log(`SQL(${callName}) started`);
+    }
+    let result;
+    try {
+      // Ignoring this error TS2556: Expected 3 arguments, but got 0 or more.
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      result = await fn(...args);
+    } finally {
+      sqlQueries -= 1;
+      if (allQueriesDone && sqlQueries <= 0) {
+        allQueriesDone();
+      }
+    }
+    const end = Date.now();
+    const delta = end - start;
+    if (DEBUG || delta > 10) {
+      // eslint-disable-next-line no-console
+      console.log(`SQL(${callName}) succeeded in ${end - start}ms`);
+    }
+    return result;
+  };
+}
+
+async function handleCall(
+  fn: ServerInterface[keyof ServerInterface],
+  args: Array<unknown>,
+  callName: keyof ServerInterface
+) {
+  if (!fn) {
+    throw new Error(`sql channel: ${callName} is not an available function`);
+  }
+
+  let result;
+
+  // We queue here to keep multi-query operations atomic. Without it, any multistage
+  //   data operation (even within a BEGIN/COMMIT) can become interleaved, since all
+  //   requests share one database connection.
+
+  // A needsSerial method must be run in our single concurrency queue.
+  if (fn.needsSerial) {
+    if (singleQueue) {
+      result = await singleQueue.add(makeSQLJob(fn, args, callName));
+    } else if (multipleQueue) {
+      const queue = makeNewSingleQueue();
+
+      const multipleQueueLocal = multipleQueue;
+      queue.add(() => multipleQueueLocal.onIdle());
+      multipleQueue = null;
+
+      result = await queue.add(makeSQLJob(fn, args, callName));
+    } else {
+      const queue = makeNewSingleQueue();
+      result = await queue.add(makeSQLJob(fn, args, callName));
+    }
+  } else {
+    // The request can be parallelized. To keep the same structure as the above block
+    //   we force this section into the 'lonely if' pattern.
+    // eslint-disable-next-line no-lonely-if
+    if (multipleQueue) {
+      result = await multipleQueue.add(makeSQLJob(fn, args, callName));
+    } else if (singleQueue) {
+      const queue = makeNewMultipleQueue();
+      queue.pause();
+
+      const singleQueueRef = singleQueue;
+
+      singleQueue = null;
+      const promise = queue.add(makeSQLJob(fn, args, callName));
+      if (singleQueueRef) {
+        await singleQueueRef.onIdle();
+      }
+
+      queue.start();
+      result = await promise;
+    } else {
+      const queue = makeNewMultipleQueue();
+      result = await queue.add(makeSQLJob(fn, args, callName));
+    }
+  }
+
+  return result;
+}
+
+export async function waitForPendingQueries(): Promise<void> {
+  return new Promise<void>(resolve => {
+    if (sqlQueries === 0) {
+      resolve();
+    } else {
+      allQueriesDone = () => resolve();
+    }
+  });
+}
+
+export function applyQueueing(dataInterface: ServerInterface): ServerInterface {
+  return Object.keys(dataInterface).reduce((acc, callName) => {
+    const serverInterfaceKey = callName as keyof ServerInterface;
+    acc[serverInterfaceKey] = async (...args: Array<unknown>) =>
+      handleCall(dataInterface[serverInterfaceKey], args, serverInterfaceKey);
+    return acc;
+  }, {} as ServerInterface);
+}

ts/sql/Server.ts

@@ -14,7 +14,6 @@ import mkdirp from 'mkdirp';
 import rimraf from 'rimraf';
 import PQueue from 'p-queue';
 import sql from '@journeyapps/sqlcipher';
-import { app, clipboard, dialog } from 'electron';
 
 import pify from 'pify';
 import { v4 as generateUUID } from 'uuid';
@@ -31,8 +30,6 @@ import {
   pick,
 } from 'lodash';
 
-import { redactAll } from '../../js/modules/privacy';
-import { remove as removeUserConfig } from '../../app/user_config';
 import { combineNames } from '../util/combineNames';
 
 import { GroupV2MemberType } from '../model-types.d';
@@ -54,6 +51,7 @@ import {
   StickerType,
   UnprocessedType,
 } from './Interface';
+import { applyQueueing } from './Queueing';
 
 declare global {
   // We want to extend `Function`'s properties, so we need to use an interface.
@@ -195,13 +193,14 @@ const dataInterface: ServerInterface = {
   // Server-only
 
   initialize,
+  initializeRenderer,
 
   removeKnownAttachments,
   removeKnownStickers,
   removeKnownDraftAttachments,
 };
 
-export default dataInterface;
+export default applyQueueing(dataInterface);
 
 function objectToJSON(data: any) {
   return JSON.stringify(data);
@@ -210,6 +209,14 @@ function jsonToObject(json: string): any {
   return JSON.parse(json);
 }
 
+function isRenderer() {
+  if (typeof process === 'undefined' || !process) {
+    return true;
+  }
+
+  return process.type === 'renderer';
+}
+
 async function openDatabase(filePath: string): Promise<sql.Database> {
   return new Promise((resolve, reject) => {
     let instance: sql.Database | undefined;
@@ -1702,6 +1709,7 @@ async function updateSchema(instance: PromisifiedSQLDatabase) {
 }
 
 let globalInstance: PromisifiedSQLDatabase | undefined;
+let globalInstanceRenderer: PromisifiedSQLDatabase | undefined;
 let databaseFilePath: string | undefined;
 let indexedDBPath: string | undefined;
 
@@ -1788,37 +1796,57 @@ async function initialize({
     await getMessageCount();
   } catch (error) {
     console.log('Database startup error:', error.stack);
-    const buttonIndex = dialog.showMessageBoxSync({
-      buttons: [
-        messages.copyErrorAndQuit.message,
-        messages.deleteAndRestart.message,
-      ],
-      defaultId: 0,
-      detail: redactAll(error.stack),
-      message: messages.databaseError.message,
-      noLink: true,
-      type: 'error',
-    });
-
-    if (buttonIndex === 0) {
-      clipboard.writeText(
-        `Database startup error:\n\n${redactAll(error.stack)}`
-      );
-    } else {
     if (promisified) {
       await promisified.close();
     }
-      await removeDB();
+    throw error;
-      removeUserConfig();
+  }
-      app.relaunch();
 }
 
-    app.exit(1);
+async function initializeRenderer({
-
+  configDir,
-    return false;
+  key,
+}: {
+  configDir: string;
+  key: string;
+}) {
+  if (!isRenderer()) {
+    throw new Error('Cannot call from main process.');
+  }
+  if (globalInstanceRenderer) {
+    throw new Error('Cannot initialize more than once!');
+  }
+  if (!isString(configDir)) {
+    throw new Error('initialize: configDir is required!');
+  }
+  if (!isString(key)) {
+    throw new Error('initialize: key is required!');
   }
 
-  return true;
+  if (!indexedDBPath) {
+    indexedDBPath = join(configDir, 'IndexedDB');
+  }
+
+  const dbDir = join(configDir, 'sql');
+
+  if (!databaseFilePath) {
+    databaseFilePath = join(dbDir, 'db.sqlite');
+  }
+
+  let promisified: PromisifiedSQLDatabase | undefined;
+
+  try {
+    promisified = await openAndSetUpSQLCipher(databaseFilePath, { key });
+
+    // At this point we can allow general access to the database
+    globalInstanceRenderer = promisified;
+
+    // test database
+    await getMessageCount();
+  } catch (error) {
+    window.log.error('Database startup error:', error.stack);
+    throw error;
+  }
 }
 
 async function close() {
@@ -1857,6 +1885,13 @@ async function removeIndexedDBFiles() {
 }
 
 function getInstance(): PromisifiedSQLDatabase {
+  if (isRenderer()) {
+    if (!globalInstanceRenderer) {
+      throw new Error('getInstance: globalInstanceRenderer not set!');
+    }
+    return globalInstanceRenderer;
+  }
+
   if (!globalInstance) {
     throw new Error('getInstance: globalInstance not set!');
   }
@@ -2285,6 +2320,7 @@ async function updateConversation(data: ConversationType) {
     }
   );
 }
+
 async function updateConversations(array: Array<ConversationType>) {
   const db = getInstance();
   await db.run('BEGIN TRANSACTION;');

ts/sql/initialize.ts

@@ -0,0 +1,16 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { ipcRenderer as ipc } from 'electron';
+import fs from 'fs-extra';
+import pify from 'pify';
+import sql from './Server';
+
+const getRealPath = pify(fs.realpath);
+
+export async function initialize(): Promise<void> {
+  const configDir = await getRealPath(ipc.sendSync('get-user-data-path'));
+  const key = ipc.sendSync('user-config-key');
+
+  await sql.initializeRenderer({ configDir, key });
+}

ts/test-electron/models/messages_test.ts

@@ -23,7 +23,10 @@ describe('Message', () => {
 
   function createMessage(attrs: { [key: string]: unknown }) {
     const messages = new window.Whisper.MessageCollection();
-    return messages.add(attrs);
+    return messages.add({
+      received_at: Date.now(),
+      ...attrs,
+    });
   }
 
   before(async () => {

ts/test-node/components/media-gallery/groupMessagesByDate_test.ts

@@ -17,6 +17,7 @@ const toMediaItem = (date: Date): MediaItemType => ({
   message: {
     id: 'id',
     received_at: date.getTime(),
+    received_at_ms: date.getTime(),
     attachments: [],
   },
   attachment: {
@@ -57,6 +58,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1523534400000,
+              received_at_ms: 1523534400000,
               attachments: [],
             },
             attachment: {
@@ -71,6 +73,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1523491260000,
+              received_at_ms: 1523491260000,
               attachments: [],
             },
             attachment: {
@@ -90,6 +93,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1523491140000,
+              received_at_ms: 1523491140000,
               attachments: [],
             },
             attachment: {
@@ -109,6 +113,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1523232060000,
+              received_at_ms: 1523232060000,
               attachments: [],
             },
             attachment: {
@@ -128,6 +133,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1523231940000,
+              received_at_ms: 1523231940000,
               attachments: [],
             },
             attachment: {
@@ -142,6 +148,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1522540860000,
+              received_at_ms: 1522540860000,
               attachments: [],
             },
             attachment: {
@@ -163,6 +170,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1522540740000,
+              received_at_ms: 1522540740000,
               attachments: [],
             },
             attachment: {
@@ -177,6 +185,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1519912800000,
+              received_at_ms: 1519912800000,
               attachments: [],
             },
             attachment: {
@@ -198,6 +207,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1298937540000,
+              received_at_ms: 1298937540000,
               attachments: [],
             },
             attachment: {
@@ -212,6 +222,7 @@ describe('groupMediaItemsByDate', () => {
             message: {
               id: 'id',
               received_at: 1296554400000,
+              received_at_ms: 1296554400000,
               attachments: [],
             },
             attachment: {

ts/textsecure.d.ts

@@ -23,6 +23,7 @@ export type UnprocessedType = {
   decrypted?: string;
   envelope?: string;
   id: string;
+  timestamp: number;
   serverTimestamp?: number;
   source?: string;
   sourceDevice?: number;
@@ -795,6 +796,8 @@ export declare class EnvelopeClass {
 
   // Note: these additional properties are added in the course of processing
   id: string;
+  receivedAtCounter: number;
+  receivedAtDate: number;
   unidentifiedDeliveryReceived?: boolean;
   messageAgeSec?: number;
 }

ts/textsecure/MessageReceiver.ts

@@ -203,7 +203,7 @@ class MessageReceiverInner extends EventTarget {
     this.cacheAddBatcher = createBatcher<CacheAddItemType>({
       wait: 200,
       maxSize: 30,
-      processBatch: this.cacheAndQueueBatch.bind(this),
+      processBatch: this.cacheAndHandleBatch.bind(this),
     });
     this.cacheUpdateBatcher = createBatcher<CacheUpdateItemType>({
       wait: 500,
@@ -237,7 +237,7 @@ class MessageReceiverInner extends EventTarget {
     }
 
     // We always process our cache before processing a new websocket message
-    this.pendingQueue.add(async () => this.queueAllCached());
+    this.pendingQueue.add(async () => this.handleAllCached());
 
     this.count = 0;
     if (this.hasConnected) {
@@ -428,13 +428,16 @@ class MessageReceiverInner extends EventTarget {
           ? envelope.serverTimestamp.toNumber()
           : null;
 
+        envelope.receivedAtCounter = window.Signal.Util.incrementMessageCounter();
+        envelope.receivedAtDate = Date.now();
+
         // Calculate the message age (time on server).
         envelope.messageAgeSec = this.calculateMessageAge(
           headers,
           envelope.serverTimestamp
         );
 
-        this.cacheAndQueue(envelope, plaintext, request);
+        this.cacheAndHandle(envelope, plaintext, request);
       } catch (e) {
         request.respond(500, 'Bad encrypted websocket message');
         window.log.error(
@@ -553,16 +556,17 @@ class MessageReceiverInner extends EventTarget {
     this.dispatchEvent(ev);
   }
 
-  async queueAllCached() {
+  async handleAllCached() {
     const items = await this.getAllFromCache();
     const max = items.length;
     for (let i = 0; i < max; i += 1) {
       // eslint-disable-next-line no-await-in-loop
-      await this.queueCached(items[i]);
+      await this.handleCachedEnvelope(items[i]);
     }
   }
 
-  async queueCached(item: UnprocessedType) {
+  async handleCachedEnvelope(item: UnprocessedType) {
+    window.log.info('MessageReceiver.handleCachedEnvelope', item.id);
     try {
       let envelopePlaintext: ArrayBuffer;
 
@@ -576,7 +580,7 @@ class MessageReceiverInner extends EventTarget {
         );
       } else {
         throw new Error(
-          'MessageReceiver.queueCached: item.envelope was malformed'
+          'MessageReceiver.handleCachedEnvelope: item.envelope was malformed'
         );
       }
 
@@ -584,6 +588,8 @@ class MessageReceiverInner extends EventTarget {
         envelopePlaintext
       );
       envelope.id = item.id;
+      envelope.receivedAtCounter = item.timestamp;
+      envelope.receivedAtDate = Date.now();
       envelope.source = envelope.source || item.source;
       envelope.sourceUuid = envelope.sourceUuid || item.sourceUuid;
       envelope.sourceDevice = envelope.sourceDevice || item.sourceDevice;
@@ -605,13 +611,13 @@ class MessageReceiverInner extends EventTarget {
         } else {
           throw new Error('Cached decrypted value was not a string!');
         }
-        this.queueDecryptedEnvelope(envelope, payloadPlaintext);
+        this.handleDecryptedEnvelope(envelope, payloadPlaintext);
       } else {
-        this.queueEnvelope(envelope);
+        this.handleEnvelope(envelope);
       }
     } catch (error) {
       window.log.error(
-        'queueCached error handling item',
+        'handleCachedEnvelope error handling item',
         item.id,
         'removing it. Error:',
         error && error.stack ? error.stack : error
@@ -622,7 +628,7 @@ class MessageReceiverInner extends EventTarget {
         await window.textsecure.storage.unprocessed.remove(id);
       } catch (deleteError) {
         window.log.error(
-          'queueCached error deleting item',
+          'handleCachedEnvelope error deleting item',
           item.id,
           'Error:',
           deleteError && deleteError.stack ? deleteError.stack : deleteError
@@ -656,7 +662,7 @@ class MessageReceiverInner extends EventTarget {
     if (this.isEmptied) {
       this.clearRetryTimeout();
       this.retryCachedTimeout = setTimeout(() => {
-        this.pendingQueue.add(async () => this.queueAllCached());
+        this.pendingQueue.add(async () => this.handleAllCached());
       }, RETRY_TIMEOUT);
     }
   }
@@ -705,7 +711,8 @@ class MessageReceiverInner extends EventTarget {
     );
   }
 
-  async cacheAndQueueBatch(items: Array<CacheAddItemType>) {
+  async cacheAndHandleBatch(items: Array<CacheAddItemType>) {
+    window.log.info('MessageReceiver.cacheAndHandleBatch', items.length);
     const dataArray = items.map(item => item.data);
     try {
       await window.textsecure.storage.unprocessed.batchAdd(dataArray);
@@ -714,16 +721,16 @@ class MessageReceiverInner extends EventTarget {
           item.request.respond(200, 'OK');
         } catch (error) {
           window.log.error(
-            'cacheAndQueueBatch: Failed to send 200 to server; still queuing envelope'
+            'cacheAndHandleBatch: Failed to send 200 to server; still queuing envelope'
           );
         }
-        this.queueEnvelope(item.envelope);
+        this.handleEnvelope(item.envelope);
       });
 
       this.maybeScheduleRetryTimeout();
     } catch (error) {
       window.log.error(
-        'cacheAndQueue error trying to add messages to cache:',
+        'cacheAndHandleBatch error trying to add messages to cache:',
         error && error.stack ? error.stack : error
       );
 
@@ -733,7 +740,7 @@ class MessageReceiverInner extends EventTarget {
     }
   }
 
-  cacheAndQueue(
+  cacheAndHandle(
     envelope: EnvelopeClass,
     plaintext: ArrayBuffer,
     request: IncomingWebSocketRequest
@@ -743,7 +750,7 @@ class MessageReceiverInner extends EventTarget {
       id,
       version: 2,
       envelope: MessageReceiverInner.arrayBufferToStringBase64(plaintext),
-      timestamp: Date.now(),
+      timestamp: envelope.receivedAtCounter,
       attempts: 1,
     };
     this.cacheAddBatcher.add({
@@ -754,6 +761,7 @@ class MessageReceiverInner extends EventTarget {
   }
 
   async cacheUpdateBatch(items: Array<Partial<UnprocessedType>>) {
+    window.log.info('MessageReceiver.cacheUpdateBatch', items.length);
     await window.textsecure.storage.unprocessed.addDecryptedDataToList(items);
   }
 
@@ -778,56 +786,6 @@ class MessageReceiverInner extends EventTarget {
     this.cacheRemoveBatcher.add(id);
   }
 
-  async queueDecryptedEnvelope(
-    envelope: EnvelopeClass,
-    plaintext: ArrayBuffer
-  ) {
-    const id = this.getEnvelopeId(envelope);
-    window.log.info('queueing decrypted envelope', id);
-
-    const task = this.handleDecryptedEnvelope.bind(this, envelope, plaintext);
-    const taskWithTimeout = window.textsecure.createTaskWithTimeout(
-      task,
-      `queueEncryptedEnvelope ${id}`
-    );
-    const promise = this.addToQueue(taskWithTimeout);
-
-    return promise.catch(error => {
-      window.log.error(
-        `queueDecryptedEnvelope error handling envelope ${id}:`,
-        error && error.extra ? JSON.stringify(error.extra) : '',
-        error && error.stack ? error.stack : error
-      );
-    });
-  }
-
-  async queueEnvelope(envelope: EnvelopeClass) {
-    const id = this.getEnvelopeId(envelope);
-    window.log.info('queueing envelope', id);
-
-    const task = this.handleEnvelope.bind(this, envelope);
-    const taskWithTimeout = window.textsecure.createTaskWithTimeout(
-      task,
-      `queueEnvelope ${id}`
-    );
-    const promise = this.addToQueue(taskWithTimeout);
-
-    return promise.catch(error => {
-      const args = [
-        'queueEnvelope error handling envelope',
-        this.getEnvelopeId(envelope),
-        ':',
-        error && error.extra ? JSON.stringify(error.extra) : '',
-        error && error.stack ? error.stack : error,
-      ];
-      if (error.warn) {
-        window.log.warn(...args);
-      } else {
-        window.log.error(...args);
-      }
-    });
-  }
-
   // Same as handleEnvelope, just without the decryption step. Necessary for handling
   //   messages which were successfully decrypted, but application logic didn't finish
   //   processing.
@@ -835,6 +793,10 @@ class MessageReceiverInner extends EventTarget {
     envelope: EnvelopeClass,
     plaintext: ArrayBuffer
   ): Promise<void> {
+    const id = this.getEnvelopeId(envelope);
+    window.log.info('MessageReceiver.handleDecryptedEnvelope', id);
+
+    try {
       if (this.stoppingProcessing) {
         return;
       }
@@ -854,9 +816,20 @@ class MessageReceiverInner extends EventTarget {
 
       this.removeFromCache(envelope);
       throw new Error('Received message with no content and no legacyMessage');
+    } catch (error) {
+      window.log.error(
+        `handleDecryptedEnvelope error handling envelope ${id}:`,
+        error && error.extra ? JSON.stringify(error.extra) : '',
+        error && error.stack ? error.stack : error
+      );
+    }
   }
 
   async handleEnvelope(envelope: EnvelopeClass) {
+    const id = this.getEnvelopeId(envelope);
+    window.log.info('MessageReceiver.handleEnvelope', id);
+
+    try {
       if (this.stoppingProcessing) {
         return Promise.resolve();
       }
@@ -871,8 +844,26 @@ class MessageReceiverInner extends EventTarget {
       if (envelope.legacyMessage) {
         return this.handleLegacyMessage(envelope);
       }
+
       this.removeFromCache(envelope);
+
       throw new Error('Received message with no content and no legacyMessage');
+    } catch (error) {
+      const args = [
+        'handleEnvelope error handling envelope',
+        this.getEnvelopeId(envelope),
+        ':',
+        error && error.extra ? JSON.stringify(error.extra) : '',
+        error && error.stack ? error.stack : error,
+      ];
+      if (error.warn) {
+        window.log.warn(...args);
+      } else {
+        window.log.error(...args);
+      }
+    }
+
+    return undefined;
   }
 
   getStatus() {
@@ -1257,6 +1248,10 @@ class MessageReceiverInner extends EventTarget {
     envelope: EnvelopeClass,
     sentContainer: SyncMessageClass.Sent
   ) {
+    window.log.info(
+      'MessageReceiver.handleSentMessage',
+      this.getEnvelopeId(envelope)
+    );
     const {
       destination,
       destinationUuid,
@@ -1324,6 +1319,8 @@ class MessageReceiverInner extends EventTarget {
           unidentifiedStatus,
           message,
           isRecipientUpdate,
+          receivedAtCounter: envelope.receivedAtCounter,
+          receivedAtDate: envelope.receivedAtDate,
         };
         if (expirationStartTimestamp) {
           ev.data.expirationStartTimestamp = expirationStartTimestamp.toNumber();
@@ -1334,7 +1331,10 @@ class MessageReceiverInner extends EventTarget {
   }
 
   async handleDataMessage(envelope: EnvelopeClass, msg: DataMessageClass) {
-    window.log.info('data message from', this.getEnvelopeId(envelope));
+    window.log.info(
+      'MessageReceiver.handleDataMessage',
+      this.getEnvelopeId(envelope)
+    );
     let p: Promise<any> = Promise.resolve();
     // eslint-disable-next-line no-bitwise
     const destination = envelope.sourceUuid || envelope.source;
@@ -1412,6 +1412,8 @@ class MessageReceiverInner extends EventTarget {
           serverTimestamp: envelope.serverTimestamp,
           unidentifiedDeliveryReceived: envelope.unidentifiedDeliveryReceived,
           message,
+          receivedAtCounter: envelope.receivedAtCounter,
+          receivedAtDate: envelope.receivedAtDate,
         };
         return this.dispatchAndWait(ev);
       })
@@ -1419,6 +1421,10 @@ class MessageReceiverInner extends EventTarget {
   }
 
   async handleLegacyMessage(envelope: EnvelopeClass) {
+    window.log.info(
+      'MessageReceiver.handleLegacyMessage',
+      this.getEnvelopeId(envelope)
+    );
     return this.decrypt(envelope, envelope.legacyMessage).then(plaintext => {
       if (!plaintext) {
         window.log.warn('handleLegacyMessage: plaintext was falsey');
@@ -1437,6 +1443,10 @@ class MessageReceiverInner extends EventTarget {
   }
 
   async handleContentMessage(envelope: EnvelopeClass) {
+    window.log.info(
+      'MessageReceiver.handleContentMessage',
+      this.getEnvelopeId(envelope)
+    );
     return this.decrypt(envelope, envelope.content).then(plaintext => {
       if (!plaintext) {
         window.log.warn('handleContentMessage: plaintext was falsey');
@@ -1579,7 +1589,10 @@ class MessageReceiverInner extends EventTarget {
   }
 
   handleNullMessage(envelope: EnvelopeClass) {
-    window.log.info('null message from', this.getEnvelopeId(envelope));
+    window.log.info(
+      'MessageReceiver.handleNullMessage',
+      this.getEnvelopeId(envelope)
+    );
     this.removeFromCache(envelope);
   }
 
@@ -1778,7 +1791,6 @@ class MessageReceiverInner extends EventTarget {
       return undefined;
     }
     if (syncMessage.read && syncMessage.read.length) {
-      window.log.info('read messages from', this.getEnvelopeId(envelope));
       return this.handleRead(envelope, syncMessage.read);
     }
     if (syncMessage.verified) {
@@ -1952,6 +1964,7 @@ class MessageReceiverInner extends EventTarget {
     envelope: EnvelopeClass,
     read: Array<SyncMessageClass.Read>
   ) {
+    window.log.info('MessageReceiver.handleRead', this.getEnvelopeId(envelope));
     const results = [];
     for (let i = 0; i < read.length; i += 1) {
       const ev = new Event('readSync');

ts/util/getMessageTimestamp.ts

@@ -0,0 +1,8 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { Message } from '../components/conversation/media-gallery/types/Message';
+
+export function getMessageTimestamp(message: Message): number {
+  return message.received_at_ms || message.received_at;
+}

ts/util/incrementMessageCounter.ts

@@ -0,0 +1,23 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { debounce } from 'lodash';
+
+export function incrementMessageCounter(): number {
+  if (!window.receivedAtCounter) {
+    window.receivedAtCounter =
+      Number(localStorage.getItem('lastReceivedAtCounter')) || Date.now();
+  }
+
+  window.receivedAtCounter += 1;
+  debouncedUpdateLastReceivedAt();
+
+  return window.receivedAtCounter;
+}
+
+const debouncedUpdateLastReceivedAt = debounce(() => {
+  localStorage.setItem(
+    'lastReceivedAtCounter',
+    String(window.receivedAtCounter)
+  );
+}, 500);

ts/util/index.ts

@@ -14,8 +14,10 @@ import { getStringForProfileChange } from './getStringForProfileChange';
 import { getTextWithMentions } from './getTextWithMentions';
 import { getUserAgent } from './getUserAgent';
 import { hasExpired } from './hasExpired';
+import { incrementMessageCounter } from './incrementMessageCounter';
 import { isFileDangerous } from './isFileDangerous';
 import { makeLookup } from './makeLookup';
+import { saveNewMessageBatcher, updateMessageBatcher } from './messageBatcher';
 import { missingCaseError } from './missingCaseError';
 import { parseRemoteClientExpiration } from './parseRemoteClientExpiration';
 import { sleep } from './sleep';
@@ -29,6 +31,8 @@ import {
 import * as zkgroup from './zkgroup';
 
 export {
+  GoogleChrome,
+  Registration,
   arrayBufferToObjectURL,
   combineNames,
   createBatcher,
@@ -40,18 +44,19 @@ export {
   getStringForProfileChange,
   getTextWithMentions,
   getUserAgent,
-  GoogleChrome,
   hasExpired,
+  incrementMessageCounter,
   isFileDangerous,
   longRunningTaskWrapper,
   makeLookup,
   mapToSupportLocale,
   missingCaseError,
   parseRemoteClientExpiration,
-  Registration,
+  saveNewMessageBatcher,
   sessionRecordToProtobuf,
   sessionStructureToArrayBuffer,
   sleep,
   toWebSafeBase64,
+  updateMessageBatcher,
   zkgroup,
 };

ts/util/lint/exceptions.json

@@ -15022,7 +15022,7 @@
     "rule": "React-createRef",
     "path": "ts/components/conversation/media-gallery/MediaGallery.js",
     "line": "        this.focusRef = react_1.default.createRef();",
-    "lineNumber": 31,
+    "lineNumber": 32,
     "reasonCategory": "usageTrusted",
     "updated": "2019-11-01T22:46:33.013Z",
     "reasonDetail": "Used for setting focus only"
@@ -15031,7 +15031,7 @@
     "rule": "React-createRef",
     "path": "ts/components/conversation/media-gallery/MediaGallery.tsx",
     "line": "  public readonly focusRef: React.RefObject<HTMLDivElement> = React.createRef();",
-    "lineNumber": 71,
+    "lineNumber": 72,
     "reasonCategory": "usageTrusted",
     "updated": "2019-11-01T22:46:33.013Z",
     "reasonDetail": "Used for setting focus only"

ts/util/messageBatcher.ts

@@ -0,0 +1,24 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { MessageAttributesType } from '../model-types.d';
+import { createBatcher } from './batcher';
+import { createWaitBatcher } from './waitBatcher';
+
+export const updateMessageBatcher = createBatcher<MessageAttributesType>({
+  wait: 500,
+  maxSize: 50,
+  processBatch: async (messages: Array<MessageAttributesType>) => {
+    window.log.info('updateMessageBatcher', messages.length);
+    await window.Signal.Data.saveMessages(messages, {});
+  },
+});
+
+export const saveNewMessageBatcher = createWaitBatcher<MessageAttributesType>({
+  wait: 500,
+  maxSize: 30,
+  processBatch: async (messages: Array<MessageAttributesType>) => {
+    window.log.info('saveNewMessageBatcher', messages.length);
+    await window.Signal.Data.saveMessages(messages, { forceSave: true });
+  },
+});

ts/window.d.ts

@@ -137,10 +137,12 @@ declare global {
 
     WhatIsThis: WhatIsThis;
 
+    attachmentDownloadQueue: Array<MessageModel>;
     baseAttachmentsPath: string;
     baseStickersPath: string;
     baseTempPath: string;
     dcodeIO: DCodeIOType;
+    receivedAtCounter: number;
     enterKeyboardMode: () => void;
     enterMouseMode: () => void;
     getAccountManager: () => AccountManager | undefined;
@@ -246,6 +248,9 @@ declare global {
     titleBarDoubleClick: () => void;
     unregisterForActive: (handler: () => void) => void;
     updateTrayIcon: (count: number) => void;
+    sqlInitializer: {
+      initialize: () => Promise<void>;
+    };
 
     Backbone: typeof Backbone;
     Signal: {
@@ -561,6 +566,8 @@ export type DCodeIOType = {
 };
 
 type MessageControllerType = {
+  findBySender: (sender: string) => MessageModel | null;
+  findBySentAt: (sentAt: number) => MessageModel | null;
   register: (id: string, model: MessageModel) => MessageModel;
   unregister: (id: string) => void;
 };