Optimize loading stories

ts/services/storyLoader.ts

@@ -7,6 +7,7 @@ import type { StoryDataType } from '../state/ducks/stories';
 import * as durations from '../util/durations';
 import * as log from '../logging/log';
 import dataInterface from '../sql/Client';
+import type { GetAllStoriesResultType } from '../sql/Interface';
 import {
   getAttachmentsForMessage,
   getPropsForAttachment,
@@ -18,32 +19,10 @@ import { dropNull } from '../util/dropNull';
 import { DurationInSeconds } from '../util/durations';
 import { SIGNAL_ACI } from '../types/SignalConversation';
 
-let storyData:
+let storyData: GetAllStoriesResultType | undefined;
-  | Array<
-      MessageAttributesType & {
-        hasReplies?: boolean;
-        hasRepliesFromSelf?: boolean;
-      }
-    >
-  | undefined;
 
 export async function loadStories(): Promise<void> {
-  const stories = await dataInterface.getAllStories({});
+  storyData = await dataInterface.getAllStories({});
-
-  storyData = await Promise.all(
-    stories.map(async story => {
-      const [hasReplies, hasRepliesFromSelf] = await Promise.all([
-        dataInterface.hasStoryReplies(story.id),
-        dataInterface.hasStoryRepliesFromSelf(story.id),
-      ]);
-
-      return {
-        ...story,
-        hasReplies,
-        hasRepliesFromSelf,
-      };
-    })
-  );
 
   await repairUnexpiredStories();
 }

ts/sql/Interface.ts

@@ -353,6 +353,13 @@ export type GetKnownMessageAttachmentsResultType = Readonly<{
   attachments: ReadonlyArray<string>;
 }>;
 
+export type GetAllStoriesResultType = ReadonlyArray<
+  MessageType & {
+    hasReplies: boolean;
+    hasRepliesFromSelf: boolean;
+  }
+>;
+
 export type DataInterface = {
   close: () => Promise<void>;
   removeDB: () => Promise<void>;
@@ -520,9 +527,7 @@ export type DataInterface = {
   getAllStories: (options: {
     conversationId?: string;
     sourceUuid?: UUIDStringType;
-  }) => Promise<Array<MessageType>>;
+  }) => Promise<GetAllStoriesResultType>;
-  hasStoryReplies: (storyId: string) => Promise<boolean>;
-  hasStoryRepliesFromSelf: (storyId: string) => Promise<boolean>;
   // getNewerMessagesByConversation is JSON on server, full message on Client
   getMessageMetricsForConversation: (
     conversationId: string,

ts/sql/Server.ts

@@ -78,6 +78,7 @@ import type {
   DeleteSentProtoRecipientOptionsType,
   DeleteSentProtoRecipientResultType,
   EmojiType,
+  GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
   GetUnreadByConversationAndMarkReadResultType,
@@ -248,8 +249,6 @@ const dataInterface: ServerInterface = {
   getTapToViewMessagesNeedingErase,
   getOlderMessagesByConversation,
   getAllStories,
-  hasStoryReplies,
-  hasStoryRepliesFromSelf,
   getNewerMessagesByConversation,
   getTotalUnreadForConversation,
   getMessageMetricsForConversation,
@@ -1770,22 +1769,21 @@ async function getMessageCount(conversationId?: string): Promise<number> {
 function hasUserInitiatedMessages(conversationId: string): boolean {
   const db = getInstance();
 
-  const row: { count: number } = db
+  const exists: number = db
     .prepare<Query>(
       `
-      SELECT COUNT(*) as count FROM
+      SELECT EXISTS(
-        (
+        SELECT 1 FROM messages
-          SELECT 1 FROM messages
+        WHERE
-          WHERE
+          conversationId = $conversationId AND
-            conversationId = $conversationId AND
+          isUserInitiatedMessage = 1
-            isUserInitiatedMessage = 1
+      );
-          LIMIT 1
-        );
       `
     )
+    .pluck()
     .get({ conversationId });
 
-  return row.count !== 0;
+  return exists !== 0;
 }
 
 function saveMessageSync(
@@ -2515,12 +2513,29 @@ async function getAllStories({
 }: {
   conversationId?: string;
   sourceUuid?: UUIDStringType;
-}): Promise<Array<MessageType>> {
+}): Promise<GetAllStoriesResultType> {
   const db = getInstance();
-  const rows: JSONRows = db
+  const rows: ReadonlyArray<{
+    json: string;
+    hasReplies: number;
+    hasRepliesFromSelf: number;
+  }> = db
     .prepare<Query>(
       `
-      SELECT json
+      SELECT
+        json,
+        (SELECT EXISTS(
+          SELECT 1
+          FROM messages as replies
+          WHERE replies.storyId IS messages.id
+        )) as hasReplies,
+        (SELECT EXISTS(
+          SELECT 1
+          FROM messages AS selfReplies
+          WHERE
+            selfReplies.storyId IS messages.id AND
+            selfReplies.type IS 'outgoing'
+        )) as hasRepliesFromSelf
       FROM messages
       WHERE
         type IS 'story' AND
@@ -2534,39 +2549,11 @@ async function getAllStories({
       sourceUuid: sourceUuid || null,
     });
 
-  return rows.map(row => jsonToObject(row.json));
+  return rows.map(row => ({
-}
+    ...jsonToObject(row.json),
-
+    hasReplies: row.hasReplies !== 0,
-async function hasStoryReplies(storyId: string): Promise<boolean> {
+    hasRepliesFromSelf: row.hasRepliesFromSelf !== 0,
-  const db = getInstance();
+  }));
-
-  const row: { count: number } = db
-    .prepare<Query>(
-      `
-      SELECT COUNT(*) as count
-      FROM messages
-      WHERE storyId IS $storyId;
-      `
-    )
-    .get({ storyId });
-
-  return row.count !== 0;
-}
-
-async function hasStoryRepliesFromSelf(storyId: string): Promise<boolean> {
-  const db = getInstance();
-
-  const sql = `
-  SELECT COUNT(*) as count
-  FROM messages
-  WHERE
-    storyId IS $storyId AND
-    type IS 'outgoing'
-  `;
-
-  const row: { count: number } = db.prepare<Query>(sql).get({ storyId });
-
-  return row.count !== 0;
 }
 
 async function getNewerMessagesByConversation(
@@ -3016,26 +3003,25 @@ async function hasGroupCallHistoryMessage(
 ): Promise<boolean> {
   const db = getInstance();
 
-  const row: { 'count(*)': number } | undefined = db
+  const exists: number = db
     .prepare<Query>(
       `
-      SELECT count(*) FROM messages
+      SELECT EXISTS(
-      WHERE conversationId = $conversationId
+        SELECT 1 FROM messages
-      AND type = 'call-history'
+        WHERE conversationId = $conversationId
-      AND json_extract(json, '$.callHistoryDetails.callMode') = 'Group'
+        AND type = 'call-history'
-      AND json_extract(json, '$.callHistoryDetails.eraId') = $eraId
+        AND json_extract(json, '$.callHistoryDetails.callMode') = 'Group'
-      LIMIT 1;
+        AND json_extract(json, '$.callHistoryDetails.eraId') = $eraId
+      );
       `
     )
+    .pluck()
     .get({
       conversationId,
       eraId,
     });
 
-  if (row) {
+  return exists !== 0;
-    return Boolean(row['count(*)']);
-  }
-  return false;
 }
 
 async function migrateConversationMessages(

ts/sql/migrations/70-story-reply-index.ts

@@ -0,0 +1,29 @@
+// Copyright 2021-2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from 'better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export default function updateToSchemaVersion70(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 70) {
+    return;
+  }
+
+  db.transaction(() => {
+    // Used in `getAllStories`.
+    db.exec(
+      `
+      CREATE INDEX messages_by_storyId ON messages (storyId);
+      `
+    );
+
+    db.pragma('user_version = 70');
+  })();
+
+  logger.info('updateToSchemaVersion70: success!');
+}

ts/sql/migrations/index.ts

@@ -45,6 +45,7 @@ import updateToSchemaVersion66 from './66-add-pni-signature-to-sent-protos';
 import updateToSchemaVersion67 from './67-add-story-to-unprocessed';
 import updateToSchemaVersion68 from './68-drop-deprecated-columns';
 import updateToSchemaVersion69 from './69-group-call-ring-cancellations';
+import updateToSchemaVersion70 from './70-story-reply-index';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -1952,6 +1953,7 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion67,
   updateToSchemaVersion68,
   updateToSchemaVersion69,
+  updateToSchemaVersion70,
 ];
 
 export function updateSchema(db: Database, logger: LoggerType): void {