Optimize a few queries

2021-12-20 13:04:02 -08:00 · 2021-12-20 13:04:02 -08:00 · 60a53656af
commit 60a53656af
parent b08691b35b
27 changed files with 1288 additions and 186 deletions
--- a/ts/sql/Client.ts
+++ b/ts/sql/Client.ts
@ -1076,7 +1076,11 @@ async function getMessageCount(conversationId?: string) {

 async function saveMessage(
  data: MessageType,
-  options: { jobToInsert?: Readonly<StoredJob>; forceSave?: boolean } = {}
+  options: {
+    jobToInsert?: Readonly<StoredJob>;
+    forceSave?: boolean;
+    ourUuid: UUIDStringType;
+  }
 ) {
  const id = await channels.saveMessage(_cleanMessageData(data), {
    ...options,
@ -1091,7 +1095,7 @@ async function saveMessage(

 async function saveMessages(
  arrayOfMessages: Array<MessageType>,
-  options?: { forceSave?: boolean }
+  options: { forceSave?: boolean; ourUuid: UUIDStringType }
 ) {
  await channels.saveMessages(
    arrayOfMessages.map(message => _cleanMessageData(message)),
--- a/ts/sql/Interface.ts
+++ b/ts/sql/Interface.ts
@ -371,14 +371,15 @@ export type DataInterface = {
  getMessageCount: (conversationId?: string) => Promise<number>;
  saveMessage: (
    data: MessageType,
-    options?: {
+    options: {
      jobToInsert?: StoredJob;
      forceSave?: boolean;
+      ourUuid: UUIDStringType;
    }
  ) => Promise<string>;
  saveMessages: (
    arrayOfMessages: Array<MessageType>,
-    options?: { forceSave?: boolean }
+    options: { forceSave?: boolean; ourUuid: UUIDStringType }
  ) => Promise<void>;
  removeMessage: (id: string) => Promise<void>;
  removeMessages: (ids: Array<string>) => Promise<void>;
--- a/ts/sql/Server.ts
+++ b/ts/sql/Server.ts
@ -1688,20 +1688,7 @@ function hasUserInitiatedMessages(conversationId: string): boolean {
          SELECT 1 FROM messages
          WHERE
            conversationId = $conversationId AND
-            (type IS NULL
-              OR
-              type NOT IN (
-                'change-number-notification',
-                'group-v1-migration',
-                'group-v2-change',
-                'keychange',
-                'message-history-unsynced',
-                'profile-change',
-                'story',
-                'universal-timer-notification',
-                'verified-change'
-              )
-            )
+            isUserInitiatedMessage = 1
          LIMIT 1
        );
      `
@ -1714,17 +1701,19 @@ function hasUserInitiatedMessages(conversationId: string): boolean {
 function saveMessageSync(
  data: MessageType,
  options: {
-    jobToInsert?: StoredJob;
-    forceSave?: boolean;
    alreadyInTransaction?: boolean;
    db?: Database;
-  } = {}
+    forceSave?: boolean;
+    jobToInsert?: StoredJob;
+    ourUuid: UUIDStringType;
+  }
 ): string {
  const {
-    jobToInsert,
-    forceSave,
    alreadyInTransaction,
    db = getInstance(),
+    forceSave,
+    jobToInsert,
+    ourUuid,
  } = options;

  if (!alreadyInTransaction) {
@ -1741,6 +1730,7 @@ function saveMessageSync(
  const {
    body,
    conversationId,
+    groupV2Change,
    hasAttachments,
    hasFileAttachments,
    hasVisualMediaAttachments,
@ -1772,6 +1762,7 @@ function saveMessageSync(
    hasAttachments: hasAttachments ? 1 : 0,
    hasFileAttachments: hasFileAttachments ? 1 : 0,
    hasVisualMediaAttachments: hasVisualMediaAttachments ? 1 : 0,
+    isChangeCreatedByUs: groupV2Change?.from === ourUuid ? 1 : 0,
    isErased: isErased ? 1 : 0,
    isViewOnce: isViewOnce ? 1 : 0,
    received_at: received_at || null,
@ -1801,6 +1792,7 @@ function saveMessageSync(
        hasAttachments = $hasAttachments,
        hasFileAttachments = $hasFileAttachments,
        hasVisualMediaAttachments = $hasVisualMediaAttachments,
+        isChangeCreatedByUs = $isChangeCreatedByUs,
        isErased = $isErased,
        isViewOnce = $isViewOnce,
        received_at = $received_at,
@ -1843,6 +1835,7 @@ function saveMessageSync(
      hasAttachments,
      hasFileAttachments,
      hasVisualMediaAttachments,
+      isChangeCreatedByUs,
      isErased,
      isViewOnce,
      received_at,
@ -1866,6 +1859,7 @@ function saveMessageSync(
      $hasAttachments,
      $hasFileAttachments,
      $hasVisualMediaAttachments,
+      $isChangeCreatedByUs,
      $isErased,
      $isViewOnce,
      $received_at,
@ -1895,10 +1889,11 @@ function saveMessageSync(

 async function saveMessage(
  data: MessageType,
-  options?: {
+  options: {
    jobToInsert?: StoredJob;
    forceSave?: boolean;
    alreadyInTransaction?: boolean;
+    ourUuid: UUIDStringType;
  }
 ): Promise<string> {
  return saveMessageSync(data, options);
@ -1906,15 +1901,14 @@ async function saveMessage(

 async function saveMessages(
  arrayOfMessages: Array<MessageType>,
-  options?: { forceSave?: boolean }
+  options: { forceSave?: boolean; ourUuid: UUIDStringType }
 ): Promise<void> {
  const db = getInstance();
-  const { forceSave } = options || {};

  db.transaction(() => {
    for (const message of arrayOfMessages) {
      assertSync(
-        saveMessageSync(message, { forceSave, alreadyInTransaction: true })
+        saveMessageSync(message, { ...options, alreadyInTransaction: true })
      );
    }
  })();
@ -2070,7 +2064,7 @@ async function getUnreadByConversationAndMarkRead({
          expirationStartTimestamp IS NULL OR
          expirationStartTimestamp > $expirationStartTimestamp
        ) AND
-        expireTimer IS NOT NULL AND
+        expireTimer > 0 AND
        conversationId = $conversationId AND
        storyId IS $storyId AND
        received_at <= $newestUnreadAt;
@ -2161,7 +2155,7 @@ async function getUnreadReactionsAndMarkRead({
        FROM reactions
        JOIN messages on messages.id IS reactions.messageId
        WHERE
-          unread IS NOT 0 AND
+          unread > 0 AND
          messages.conversationId IS $conversationId AND
          messages.received_at <= $newestUnreadAt AND
          messages.storyId IS $storyId
@ -2504,31 +2498,9 @@ function getLastConversationActivity({
      SELECT json FROM messages
      WHERE
        conversationId = $conversationId AND
-        (type IS NULL
-          OR
-          type NOT IN (
-            'change-number-notification',
-            'group-v1-migration',
-            'keychange',
-            'message-history-unsynced',
-            'profile-change',
-            'story',
-            'universal-timer-notification',
-            'verified-change'
-          )
-        ) AND
-        (
-          json_extract(json, '$.expirationTimerUpdate.fromSync') IS NULL
-          OR
-          json_extract(json, '$.expirationTimerUpdate.fromSync') != 1
-        ) AND NOT
-        (
-          type IS 'group-v2-change' AND
-          json_extract(json, '$.groupV2Change.from') IS NOT $ourUuid AND
-          json_extract(json, '$.groupV2Change.details.length') IS 1 AND
-          json_extract(json, '$.groupV2Change.details[0].type') IS 'member-remove' AND
-          json_extract(json, '$.groupV2Change.details[0].uuid') IS NOT $ourUuid
-        )
+        shouldAffectActivity IS 1 AND
+        isTimerChangeFromSync IS 0 AND
+        isGroupLeaveEventFromOther IS 0
      ORDER BY received_at DESC, sent_at DESC
      LIMIT 1;
      `
@ -2557,29 +2529,12 @@ function getLastConversationPreview({
      SELECT json FROM messages
      WHERE
        conversationId = $conversationId AND
+        shouldAffectPreview IS 1 AND
+        isGroupLeaveEventFromOther IS 0 AND
        (
-          expiresAt IS NULL OR
-          (expiresAt > $now)
-        ) AND
-        (
-          type IS NULL
+          expiresAt IS NULL
          OR
-          type NOT IN (
-            'change-number-notification',
-            'group-v1-migration',
-            'message-history-unsynced',
-            'profile-change',
-            'story',
-            'universal-timer-notification',
-            'verified-change'
-          )
-        ) AND NOT
-        (
-          type IS 'group-v2-change' AND
-          json_extract(json, '$.groupV2Change.from') IS NOT $ourUuid AND
-          json_extract(json, '$.groupV2Change.details.length') IS 1 AND
-          json_extract(json, '$.groupV2Change.details[0].type') IS 'member-remove' AND
-          json_extract(json, '$.groupV2Change.details[0].uuid') IS NOT $ourUuid
+          expiresAt > $now
        )
      ORDER BY received_at DESC, sent_at DESC
      LIMIT 1;
--- a/ts/sql/migrations/41-uuid-keys.ts
+++ b/ts/sql/migrations/41-uuid-keys.ts
@ -11,7 +11,7 @@ import { createOrUpdate, getById, removeById } from '../util';
 import type { EmptyQuery, Query } from '../util';
 import type { ItemKeyType } from '../Interface';

-function getOurUuid(db: Database): string | undefined {
+export function getOurUuid(db: Database): string | undefined {
  const UUID_ID: ItemKeyType = 'uuid_id';

  const row: { json: string } | undefined = db
--- a/ts/sql/migrations/47-further-optimize.ts
+++ b/ts/sql/migrations/47-further-optimize.ts
@ -0,0 +1,141 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from 'better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+import { getOurUuid } from './41-uuid-keys';
+import type { Query } from '../util';
+
+export default function updateToSchemaVersion47(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 47) {
+    return;
+  }
+
+  db.transaction(() => {
+    db.exec(
+      `
+      DROP INDEX   messages_conversation;
+
+      ALTER TABLE messages
+        DROP COLUMN isStory;
+      ALTER TABLE messages
+        ADD COLUMN isStory INTEGER
+        GENERATED ALWAYS AS (type IS 'story');
+
+      ALTER TABLE messages
+        ADD COLUMN isChangeCreatedByUs INTEGER NOT NULL DEFAULT 0;
+
+      ALTER TABLE messages
+        ADD COLUMN shouldAffectActivity INTEGER
+        GENERATED ALWAYS AS (
+          type IS NULL
+          OR
+          type NOT IN (
+            'change-number-notification',
+            'group-v1-migration',
+            'message-history-unsynced',
+            'profile-change',
+            'story',
+            'universal-timer-notification',
+            'verified-change',
+
+            'keychange'
+          )
+        );
+
+      ALTER TABLE messages
+        ADD COLUMN shouldAffectPreview INTEGER
+        GENERATED ALWAYS AS (
+          type IS NULL
+          OR
+          type NOT IN (
+            'change-number-notification',
+            'group-v1-migration',
+            'message-history-unsynced',
+            'profile-change',
+            'story',
+            'universal-timer-notification',
+            'verified-change'
+          )
+        );
+
+      ALTER TABLE messages
+        ADD COLUMN isUserInitiatedMessage INTEGER
+        GENERATED ALWAYS AS (
+          type IS NULL
+          OR
+          type NOT IN (
+            'change-number-notification',
+            'group-v1-migration',
+            'message-history-unsynced',
+            'profile-change',
+            'story',
+            'universal-timer-notification',
+            'verified-change',
+
+            'group-v2-change',
+            'keychange'
+          )
+        );
+
+      ALTER TABLE messages
+        ADD COLUMN isTimerChangeFromSync INTEGER
+        GENERATED ALWAYS AS (
+          json_extract(json, '$.expirationTimerUpdate.fromSync') IS 1
+        );
+
+      ALTER TABLE messages
+        ADD COLUMN isGroupLeaveEvent INTEGER
+        GENERATED ALWAYS AS (
+          type IS 'group-v2-change' AND
+          json_array_length(json_extract(json, '$.groupV2Change.details')) IS 1 AND
+          json_extract(json, '$.groupV2Change.details[0].type') IS 'member-remove' AND
+          json_extract(json, '$.groupV2Change.from') IS NOT NULL AND
+          json_extract(json, '$.groupV2Change.from') IS json_extract(json, '$.groupV2Change.details[0].uuid')
+        );
+
+      ALTER TABLE messages
+        ADD COLUMN isGroupLeaveEventFromOther INTEGER
+        GENERATED ALWAYS AS (
+          isGroupLeaveEvent IS 1
+          AND
+          isChangeCreatedByUs IS 0
+        );
+
+      CREATE INDEX messages_conversation ON messages
+        (conversationId, isStory, storyId, received_at, sent_at);
+
+      CREATE INDEX messages_preview ON messages
+        (conversationId, shouldAffectPreview, isGroupLeaveEventFromOther, expiresAt, received_at, sent_at);
+
+      CREATE INDEX messages_activity ON messages
+        (conversationId, shouldAffectActivity, isTimerChangeFromSync, isGroupLeaveEventFromOther, received_at, sent_at);
+
+      CREATE INDEX message_user_initiated ON messages (isUserInitiatedMessage);
+      `
+    );
+
+    const ourUuid = getOurUuid(db);
+    if (!ourUuid) {
+      logger.warn('updateToSchemaVersion47: our UUID not found');
+    } else {
+      db.prepare<Query>(
+        `
+        UPDATE messages SET
+          isChangeCreatedByUs = json_extract(json, '$.groupV2Change.from') IS $ourUuid;
+        `
+      ).run({
+        ourUuid,
+      });
+    }
+
+    db.pragma('user_version = 47');
+  })();
+
+  logger.info('updateToSchemaVersion47: success!');
+}
--- a/ts/sql/migrations/index.ts
+++ b/ts/sql/migrations/index.ts
@ -22,6 +22,7 @@ import updateToSchemaVersion43 from './43-gv2-uuid';
 import updateToSchemaVersion44 from './44-badges';
 import updateToSchemaVersion45 from './45-stories';
 import updateToSchemaVersion46 from './46-optimize-stories';
+import updateToSchemaVersion47 from './47-further-optimize';

 function updateToSchemaVersion1(
  currentVersion: number,
@ -1907,6 +1908,7 @@ export const SCHEMA_VERSIONS = [
  updateToSchemaVersion44,
  updateToSchemaVersion45,
  updateToSchemaVersion46,
+  updateToSchemaVersion47,
 ];

 export function updateSchema(db: Database, logger: LoggerType): void {