From d6d53f9d18af4fd3f17e6f23391e4903e0d463bd Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Mon, 28 Nov 2022 09:19:48 -0800 Subject: [PATCH] Optimize loading stories --- ts/services/storyLoader.ts | 27 +----- ts/sql/Interface.ts | 11 ++- ts/sql/Server.ts | 104 ++++++++++------------ ts/sql/migrations/70-story-reply-index.ts | 29 ++++++ ts/sql/migrations/index.ts | 2 + 5 files changed, 87 insertions(+), 86 deletions(-) create mode 100644 ts/sql/migrations/70-story-reply-index.ts diff --git a/ts/services/storyLoader.ts b/ts/services/storyLoader.ts index 60dab944c..749d6d8dc 100644 --- a/ts/services/storyLoader.ts +++ b/ts/services/storyLoader.ts @@ -7,6 +7,7 @@ import type { StoryDataType } from '../state/ducks/stories'; import * as durations from '../util/durations'; import * as log from '../logging/log'; import dataInterface from '../sql/Client'; +import type { GetAllStoriesResultType } from '../sql/Interface'; import { getAttachmentsForMessage, getPropsForAttachment, @@ -18,32 +19,10 @@ import { dropNull } from '../util/dropNull'; import { DurationInSeconds } from '../util/durations'; import { SIGNAL_ACI } from '../types/SignalConversation'; -let storyData: - | Array< - MessageAttributesType & { - hasReplies?: boolean; - hasRepliesFromSelf?: boolean; - } - > - | undefined; +let storyData: GetAllStoriesResultType | undefined; export async function loadStories(): Promise { - const stories = await dataInterface.getAllStories({}); - - storyData = await Promise.all( - stories.map(async story => { - const [hasReplies, hasRepliesFromSelf] = await Promise.all([ - dataInterface.hasStoryReplies(story.id), - dataInterface.hasStoryRepliesFromSelf(story.id), - ]); - - return { - ...story, - hasReplies, - hasRepliesFromSelf, - }; - }) - ); + storyData = await dataInterface.getAllStories({}); await repairUnexpiredStories(); } diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 95014d394..7cc0c0d42 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -353,6 +353,13 @@ export type GetKnownMessageAttachmentsResultType = Readonly<{ attachments: ReadonlyArray; }>; +export type GetAllStoriesResultType = ReadonlyArray< + MessageType & { + hasReplies: boolean; + hasRepliesFromSelf: boolean; + } +>; + export type DataInterface = { close: () => Promise; removeDB: () => Promise; @@ -520,9 +527,7 @@ export type DataInterface = { getAllStories: (options: { conversationId?: string; sourceUuid?: UUIDStringType; - }) => Promise>; - hasStoryReplies: (storyId: string) => Promise; - hasStoryRepliesFromSelf: (storyId: string) => Promise; + }) => Promise; // getNewerMessagesByConversation is JSON on server, full message on Client getMessageMetricsForConversation: ( conversationId: string, diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 16ae3cc53..0c3a874c8 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -78,6 +78,7 @@ import type { DeleteSentProtoRecipientOptionsType, DeleteSentProtoRecipientResultType, EmojiType, + GetAllStoriesResultType, GetConversationRangeCenteredOnMessageResultType, GetKnownMessageAttachmentsResultType, GetUnreadByConversationAndMarkReadResultType, @@ -248,8 +249,6 @@ const dataInterface: ServerInterface = { getTapToViewMessagesNeedingErase, getOlderMessagesByConversation, getAllStories, - hasStoryReplies, - hasStoryRepliesFromSelf, getNewerMessagesByConversation, getTotalUnreadForConversation, getMessageMetricsForConversation, @@ -1770,22 +1769,21 @@ async function getMessageCount(conversationId?: string): Promise { function hasUserInitiatedMessages(conversationId: string): boolean { const db = getInstance(); - const row: { count: number } = db + const exists: number = db .prepare( ` - SELECT COUNT(*) as count FROM - ( - SELECT 1 FROM messages - WHERE - conversationId = $conversationId AND - isUserInitiatedMessage = 1 - LIMIT 1 - ); + SELECT EXISTS( + SELECT 1 FROM messages + WHERE + conversationId = $conversationId AND + isUserInitiatedMessage = 1 + ); ` ) + .pluck() .get({ conversationId }); - return row.count !== 0; + return exists !== 0; } function saveMessageSync( @@ -2515,12 +2513,29 @@ async function getAllStories({ }: { conversationId?: string; sourceUuid?: UUIDStringType; -}): Promise> { +}): Promise { const db = getInstance(); - const rows: JSONRows = db + const rows: ReadonlyArray<{ + json: string; + hasReplies: number; + hasRepliesFromSelf: number; + }> = db .prepare( ` - SELECT json + SELECT + json, + (SELECT EXISTS( + SELECT 1 + FROM messages as replies + WHERE replies.storyId IS messages.id + )) as hasReplies, + (SELECT EXISTS( + SELECT 1 + FROM messages AS selfReplies + WHERE + selfReplies.storyId IS messages.id AND + selfReplies.type IS 'outgoing' + )) as hasRepliesFromSelf FROM messages WHERE type IS 'story' AND @@ -2534,39 +2549,11 @@ async function getAllStories({ sourceUuid: sourceUuid || null, }); - return rows.map(row => jsonToObject(row.json)); -} - -async function hasStoryReplies(storyId: string): Promise { - const db = getInstance(); - - const row: { count: number } = db - .prepare( - ` - SELECT COUNT(*) as count - FROM messages - WHERE storyId IS $storyId; - ` - ) - .get({ storyId }); - - return row.count !== 0; -} - -async function hasStoryRepliesFromSelf(storyId: string): Promise { - const db = getInstance(); - - const sql = ` - SELECT COUNT(*) as count - FROM messages - WHERE - storyId IS $storyId AND - type IS 'outgoing' - `; - - const row: { count: number } = db.prepare(sql).get({ storyId }); - - return row.count !== 0; + return rows.map(row => ({ + ...jsonToObject(row.json), + hasReplies: row.hasReplies !== 0, + hasRepliesFromSelf: row.hasRepliesFromSelf !== 0, + })); } async function getNewerMessagesByConversation( @@ -3016,26 +3003,25 @@ async function hasGroupCallHistoryMessage( ): Promise { const db = getInstance(); - const row: { 'count(*)': number } | undefined = db + const exists: number = db .prepare( ` - SELECT count(*) FROM messages - WHERE conversationId = $conversationId - AND type = 'call-history' - AND json_extract(json, '$.callHistoryDetails.callMode') = 'Group' - AND json_extract(json, '$.callHistoryDetails.eraId') = $eraId - LIMIT 1; + SELECT EXISTS( + SELECT 1 FROM messages + WHERE conversationId = $conversationId + AND type = 'call-history' + AND json_extract(json, '$.callHistoryDetails.callMode') = 'Group' + AND json_extract(json, '$.callHistoryDetails.eraId') = $eraId + ); ` ) + .pluck() .get({ conversationId, eraId, }); - if (row) { - return Boolean(row['count(*)']); - } - return false; + return exists !== 0; } async function migrateConversationMessages( diff --git a/ts/sql/migrations/70-story-reply-index.ts b/ts/sql/migrations/70-story-reply-index.ts new file mode 100644 index 000000000..24ec09b0f --- /dev/null +++ b/ts/sql/migrations/70-story-reply-index.ts @@ -0,0 +1,29 @@ +// Copyright 2021-2022 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import type { Database } from 'better-sqlite3'; + +import type { LoggerType } from '../../types/Logging'; + +export default function updateToSchemaVersion70( + currentVersion: number, + db: Database, + logger: LoggerType +): void { + if (currentVersion >= 70) { + return; + } + + db.transaction(() => { + // Used in `getAllStories`. + db.exec( + ` + CREATE INDEX messages_by_storyId ON messages (storyId); + ` + ); + + db.pragma('user_version = 70'); + })(); + + logger.info('updateToSchemaVersion70: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index 4f8fa8318..415543d93 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -45,6 +45,7 @@ import updateToSchemaVersion66 from './66-add-pni-signature-to-sent-protos'; import updateToSchemaVersion67 from './67-add-story-to-unprocessed'; import updateToSchemaVersion68 from './68-drop-deprecated-columns'; import updateToSchemaVersion69 from './69-group-call-ring-cancellations'; +import updateToSchemaVersion70 from './70-story-reply-index'; function updateToSchemaVersion1( currentVersion: number, @@ -1952,6 +1953,7 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion67, updateToSchemaVersion68, updateToSchemaVersion69, + updateToSchemaVersion70, ]; export function updateSchema(db: Database, logger: LoggerType): void {