mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions 1

Update message attachment migration

Browse source
This commit is contained in:
trevor-signal 2025-06-02 17:16:37 -04:00 • committed by GitHub
parent a034045935
commit 115b79e4ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 97 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

2
ts/background.ts
View file

@ -1019,7 +1019,7 @@ export async function startApp(): Promise<void> {
`Starting background data migration. Target version: ${Message.CURRENT_SCHEMA_VERSION}`
);
idleDetector.on('idle', async () => {
const NUM_MESSAGES_PER_BATCH = 1000;
const NUM_MESSAGES_PER_BATCH = 250;
const BATCH_DELAY = durations.SECOND / 4;
if (isIdleTaskProcessing) {

10
ts/sql/Interface.ts
View file

@ -129,10 +129,9 @@ export type MessageType = MessageAttributesType;
// - Make sure the name matches the one in `MessageAttributeTypes`
// - Update `hydrateMessage`
//
export const MESSAGE_COLUMNS = [
const MESSAGE_PRIMARY_KEY_COLUMNS = ['id'] as const;
export const MESSAGE_NON_PRIMARY_KEY_COLUMNS = [
'json',
'id',
'body',
'conversationId',
'expirationStartTimestamp',
@ -161,6 +160,11 @@ export const MESSAGE_COLUMNS = [
'unidentifiedDeliveryReceived',
] as const;
export const MESSAGE_COLUMNS = [
...MESSAGE_PRIMARY_KEY_COLUMNS,
...MESSAGE_NON_PRIMARY_KEY_COLUMNS,
] as const;
export type MessageTypeUnhydrated = {
json: string;

25
ts/sql/Server.ts
View file

@ -191,6 +191,7 @@ import {
AttachmentDownloadSource,
MESSAGE_COLUMNS,
MESSAGE_ATTACHMENT_COLUMNS,
MESSAGE_NON_PRIMARY_KEY_COLUMNS,
} from './Interface';
import {
_removeAllCallLinks,
@ -235,6 +236,7 @@ import {
import { generateMessageId } from '../util/generateMessageId';
import type { ConversationColorType, CustomColorType } from '../types/Colors';
import { sqlLogger } from './sqlLogger';
import { APPLICATION_OCTET_STREAM } from '../types/MIME';
type ConversationRow = Readonly<{
json: string;
@ -2559,8 +2561,8 @@ function saveMessageAttachment({
conversationId,
sentAt,
clientUuid: attachment.clientUuid,
size: attachment.size,
contentType: attachment.contentType,
size: attachment.size ?? 0,
contentType: attachment.contentType ?? APPLICATION_OCTET_STREAM,
path: attachment.path,
localKey: attachment.localKey,
plaintextHash: attachment.plaintextHash,
@ -2642,8 +2644,7 @@ function saveMessageAttachment({
INSERT OR REPLACE INTO message_attachments
(${MESSAGE_ATTACHMENT_COLUMNS.join(', ')})
VALUES
(${MESSAGE_ATTACHMENT_COLUMNS.map(name => `$${name}`).join(', ')})
RETURNING rowId;
(${MESSAGE_ATTACHMENT_COLUMNS.map(name => `$${name}`).join(', ')});
`
).run(values);
}
@ -2830,20 +2831,24 @@ function saveMessage(
if (id && !forceSave) {
db.prepare(
// UPDATE queries that set the value of a primary key column can be very slow when
// that key is referenced via a foreign key constraint, so we are careful to exclude
// it here.
`
UPDATE messages SET
${MESSAGE_COLUMNS.map(name => `${name} = $${name}`).join(', ')}
WHERE id = $id;
UPDATE messages SET
${MESSAGE_NON_PRIMARY_KEY_COLUMNS.map(name => `${name} = $${name}`).join(', ')}
WHERE id = $id;
`
).run({ ...payloadWithoutJson, json: objectToJSON(dataToSaveAsJSON) });
if (normalizeAttachmentData) {
saveMessageAttachments(db, message);
}
if (jobToInsert) {
insertJob(db, jobToInsert);
}
if (normalizeAttachmentData) {
saveMessageAttachments(db, message);
}
return id;
}

44
ts/sql/migrations/1360-attachments.ts
View file

@ -85,24 +85,32 @@ export function updateToSchemaVersion1360(
) STRICT;
`);
db.exec(
'CREATE INDEX message_attachments_messageId ON message_attachments (messageId);'
);
db.exec(
'CREATE INDEX message_attachments_plaintextHash ON message_attachments (plaintextHash);'
);
db.exec(
'CREATE INDEX message_attachments_path ON message_attachments (path);'
);
db.exec(
'CREATE INDEX message_attachments_all_thumbnailPath ON message_attachments (thumbnailPath);'
);
db.exec(
'CREATE INDEX message_attachments_all_screenshotPath ON message_attachments (screenshotPath);'
);
db.exec(
'CREATE INDEX message_attachments_all_backupThumbnailPath ON message_attachments (backupThumbnailPath);'
);
// The following indexes were removed in migration 1370
// db.exec(
// 'CREATE INDEX message_attachments_messageId
// ON message_attachments (messageId);'
// );
// db.exec(
// 'CREATE INDEX message_attachments_plaintextHash
// ON message_attachments (plaintextHash);'
// );
// db.exec(
// 'CREATE INDEX message_attachments_path
// ON message_attachments (path);'
// );
// db.exec(
// 'CREATE INDEX message_attachments_all_thumbnailPath
// ON message_attachments (thumbnailPath);'
// );
// db.exec(
// 'CREATE INDEX message_attachments_all_screenshotPath
// ON message_attachments (screenshotPath);'
// );
// db.exec(
// 'CREATE INDEX message_attachments_all_backupThumbnailPath
// ON message_attachments (backupThumbnailPath);'
// );
db.pragma('user_version = 1360');
})();

31
ts/sql/migrations/1370-message-attachment-indexes.ts Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { LoggerType } from '../../types/Logging';
import type { WritableDB } from '../Interface';
export const version = 1370;
export function updateToSchemaVersion1370(
currentVersion: number,
db: WritableDB,
logger: LoggerType
): void {
if (currentVersion >= 1370) {
return;
}
db.transaction(() => {
db.exec(`
DROP INDEX IF EXISTS message_attachments_messageId;
DROP INDEX IF EXISTS message_attachments_plaintextHash;
DROP INDEX IF EXISTS message_attachments_path;
DROP INDEX IF EXISTS message_attachments_all_thumbnailPath;
DROP INDEX IF EXISTS message_attachments_all_screenshotPath;
DROP INDEX IF EXISTS message_attachments_all_backupThumbnailPath;
`);
db.pragma('user_version = 1370');
})();
logger.info('updateToSchemaVersion1370: success!');
}

6
ts/sql/migrations/index.ts
View file

@ -111,10 +111,11 @@ import { updateToSchemaVersion1320 } from './1320-unprocessed-received-at-date';
import { updateToSchemaVersion1330 } from './1330-sync-tasks-type-index';
import { updateToSchemaVersion1340 } from './1340-recent-gifs';
import { updateToSchemaVersion1350 } from './1350-notification-profiles';
import { updateToSchemaVersion1360 } from './1360-attachments';
import {
updateToSchemaVersion1360,
updateToSchemaVersion1370,
version as MAX_VERSION,
} from './1360-attachments';
} from './1370-message-attachment-indexes';
import { DataWriter } from '../Server';
@ -2104,6 +2105,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion1340,
updateToSchemaVersion1350,
updateToSchemaVersion1360,
updateToSchemaVersion1370,
];
export class DBVersionFromFutureError extends Error {

12
ts/test-electron/normalizedAttachments_test.ts
View file

@ -12,7 +12,13 @@ import type {
ThumbnailType,
BackupThumbnailType,
} from '../types/Attachment';
import { IMAGE_JPEG, IMAGE_PNG, LONG_MESSAGE } from '../types/MIME';
import {
APPLICATION_OCTET_STREAM,
IMAGE_JPEG,
IMAGE_PNG,
LONG_MESSAGE,
type MIMEType,
} from '../types/MIME';
import type { MessageAttributesType } from '../model-types';
import { generateAci } from '../types/ServiceId';
import { ReadStatus } from '../messages/MessageReadStatus';
@ -608,6 +614,8 @@ describe('normalizes attachment references', () => {
it('handles bad data', async () => {
const attachment: AttachmentType = {
...composeAttachment(),
size: undefined as unknown as number,
contentType: undefined as unknown as MIMEType,
uploadTimestamp: {
low: 6174,
high: 0,
@ -629,6 +637,8 @@ describe('normalizes attachment references', () => {
assert(messageFromDB, 'message was saved');
assert.deepEqual(messageFromDB.attachments?.[0], {
...attachment,
size: 0,
contentType: APPLICATION_OCTET_STREAM,
uploadTimestamp: undefined,
incrementalMac: undefined,
});

57
ts/test-node/sql/migration_1360_test.ts
View file

@ -3,7 +3,7 @@
import { assert } from 'chai';
import { sql, sqlJoin } from '../../sql/util';
import { sql } from '../../sql/util';
import { createDB, explain, updateToVersion } from './helpers';
import type { WritableDB } from '../../sql/Interface';
import { DataWriter } from '../../sql/Server';
@ -29,61 +29,8 @@ describe('SQL/updateToSchemaVersion1360', () => {
);
assert.strictEqual(
details,
'SEARCH message_attachments USING COVERING INDEX message_attachments_messageId (messageId=?)'
'SEARCH message_attachments USING COVERING INDEX sqlite_autoindex_message_attachments_1 (messageId=?)'
);
});
it('uses index to select based on messageId', async () => {
const details = explain(
db,
sql`SELECT * from message_attachments WHERE messageId IN (${sqlJoin(['id1', 'id2'])});`
);
assert.strictEqual(
details,
'SEARCH message_attachments USING INDEX message_attachments_messageId (messageId=?)'
);
});
it('uses index find path with existing plaintextHash', async () => {
const details = explain(
db,
sql`
SELECT path, localKey
FROM message_attachments
WHERE plaintextHash = ${'plaintextHash'}
LIMIT 1;
`
);
assert.strictEqual(
details,
'SEARCH message_attachments USING INDEX message_attachments_plaintextHash (plaintextHash=?)'
);
});
it('uses all path indices to find if path is being referenced', async () => {
const path = 'path';
const details = explain(
db,
sql`
SELECT 1 FROM message_attachments
WHERE
path = ${path} OR
thumbnailPath = ${path} OR
screenshotPath = ${path} OR
backupThumbnailPath = ${path};
`
);
assert.deepStrictEqual(details.split('\n'), [
'MULTI-INDEX OR',
'INDEX 1',
'SEARCH message_attachments USING INDEX message_attachments_path (path=?)',
'INDEX 2',
'SEARCH message_attachments USING INDEX message_attachments_all_thumbnailPath (thumbnailPath=?)',
'INDEX 3',
'SEARCH message_attachments USING INDEX message_attachments_all_screenshotPath (screenshotPath=?)',
'INDEX 4',
'SEARCH message_attachments USING INDEX message_attachments_all_backupThumbnailPath (backupThumbnailPath=?)',
]);
});
});
});