mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Normalize messages table

Browse source
This commit is contained in:
Fedor Indutny 2025-01-16 13:34:35 -08:00 committed by GitHub
parent 9bec59b70a
commit 630a1fcc89
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 490 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ts/model-types.d.ts vendored
View file

@ -168,6 +168,9 @@ type MessageType =
| 'verified-change'
| 'message-request-response-event';
// Note: when adding a property that is likely to be set across many messages,
// consider adding a database column as well and updating `MESSAGE_COLUMNS`
// in `ts/sql/Server.ts`
export type MessageAttributesType = {
bodyAttachment?: AttachmentType;
bodyRanges?: ReadonlyArray<RawBodyRange>;

9
ts/sql/Client.ts
View file

@ -62,6 +62,7 @@ import type {
ClientOnlyReadableInterface,
ClientOnlyWritableInterface,
} from './Interface';
import { hydrateMessage } from './hydration';
import type { MessageAttributesType } from '../model-types';
import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
@ -545,7 +546,7 @@ function handleSearchMessageJSON(
messages: Array<ServerSearchResultMessageType>
): Array<ClientSearchResultMessageType> {
return messages.map<ClientSearchResultMessageType>(message => {
const parsedMessage = JSON.parse(message.json);
const parsedMessage = hydrateMessage(message);
assertDev(
message.ftsSnippet ?? typeof message.mentionStart === 'number',
'Neither ftsSnippet nor matching mention returned from message search'
@ -553,14 +554,12 @@ function handleSearchMessageJSON(
const snippet =
message.ftsSnippet ??
generateSnippetAroundMention({
body: parsedMessage.body,
body: parsedMessage.body || '',
mentionStart: message.mentionStart ?? 0,
mentionLength: message.mentionLength ?? 1,
});
return {
json: message.json,
// Empty array is a default value. `message.json` has the real field
bodyRanges: [],
...parsedMessage,
@ -734,7 +733,7 @@ async function removeMessages(
function handleMessageJSON(
messages: Array<MessageTypeUnhydrated>
): Array<MessageType> {
return messages.map(message => JSON.parse(message.json));
return messages.map(message => hydrateMessage(message));
}
async function getNewerMessagesByConversation(

73
ts/sql/Interface.ts
View file

@ -115,8 +115,76 @@ export type StoredItemType<K extends ItemKeyType> = {
value: BytesToStrings<StorageAccessType[K]>;
};
export type MessageType = MessageAttributesType;
// See: ts/sql/Interface.ts
//
// When adding a new column:
//
// - Make sure the name matches the one in `MessageAttributeTypes`
// - Update `hydrateMessage`
//
export const MESSAGE_COLUMNS = [
'json',
'id',
'body',
'conversationId',
'expirationStartTimestamp',
'expireTimer',
'hasAttachments',
'hasFileAttachments',
'hasVisualMediaAttachments',
'isChangeCreatedByUs',
'isErased',
'isViewOnce',
'mentionsMe',
'received_at',
'received_at_ms',
'schemaVersion',
'serverGuid',
'sent_at',
'source',
'sourceServiceId',
'sourceDevice',
'storyId',
'type',
'readStatus',
'seenStatus',
'serverTimestamp',
'timestamp',
'unidentifiedDeliveryReceived',
] as const;
export type MessageTypeUnhydrated = {
json: string;
id: string;
body: string | null;
conversationId: string | null;
expirationStartTimestamp: number | null;
expireTimer: number | null;
hasAttachments: 0 | 1 | null;
hasFileAttachments: 0 | 1 | null;
hasVisualMediaAttachments: 0 | 1 | null;
isChangeCreatedByUs: 0 | 1 | null;
isErased: 0 | 1 | null;
isViewOnce: 0 | 1 | null;
mentionsMe: 0 | 1 | null;
received_at: number | null;
received_at_ms: number | null;
schemaVersion: number | null;
serverGuid: string | null;
sent_at: number | null;
source: string | null;
sourceServiceId: string | null;
sourceDevice: number | null;
serverTimestamp: number | null;
storyId: string | null;
type: string;
timestamp: number | null;
readStatus: number | null;
seenStatus: number | null;
unidentifiedDeliveryReceived: 0 | 1 | null;
};
export type PreKeyIdType = `${ServiceIdString}:${number}`;
@ -147,9 +215,7 @@ export type StoredPreKeyType = PreKeyType & {
privateKey: string;
publicKey: string;
};
export type ServerSearchResultMessageType = {
json: string;
export type ServerSearchResultMessageType = MessageTypeUnhydrated & {
// If the FTS matches text in message.body, snippet will be populated
ftsSnippet: string | null;
@ -159,7 +225,6 @@ export type ServerSearchResultMessageType = {
mentionLength: number | null;
};
export type ClientSearchResultMessageType = MessageType & {
json: string;
bodyRanges: ReadonlyArray<RawBodyRange>;
snippet: string;
};

334
ts/sql/Server.ts
View file

@ -55,13 +55,7 @@ import { isNormalNumber } from '../util/isNormalNumber';
import { isNotNil } from '../util/isNotNil';
import { parseIntOrThrow } from '../util/parseIntOrThrow';
import { updateSchema } from './migrations';
import type {
ArrayQuery,
EmptyQuery,
JSONRows,
Query,
QueryFragment,
} from './util';
import type { ArrayQuery, EmptyQuery, JSONRows, Query } from './util';
import {
batchMultiVarQuery,
bulkAdd,
@ -80,7 +74,9 @@ import {
sqlConstant,
sqlFragment,
sqlJoin,
QueryFragment,
} from './util';
import { hydrateMessage } from './hydration';
import { getAttachmentCiphertextLength } from '../AttachmentCrypto';
import { SeenStatus } from '../MessageSeenStatus';
@ -181,7 +177,7 @@ import type {
UnprocessedUpdateType,
WritableDB,
} from './Interface';
import { AttachmentDownloadSource } from './Interface';
import { AttachmentDownloadSource, MESSAGE_COLUMNS } from './Interface';
import {
_removeAllCallLinks,
beginDeleteAllCallLinks,
@ -582,6 +578,10 @@ export function prepare<T extends Array<unknown> | Record<string, unknown>>(
return result;
}
const MESSAGE_COLUMNS_FRAGMENTS = MESSAGE_COLUMNS.map(
column => new QueryFragment(column, [])
);
function rowToConversation(row: ConversationRow): ConversationType {
const { expireTimerVersion } = row;
const parsedJson = JSON.parse(row.json);
@ -603,6 +603,7 @@ function rowToConversation(row: ConversationRow): ConversationType {
profileLastFetchedAt,
};
}
function rowToSticker(row: StickerRow): StickerType {
return {
...row,
@ -1938,6 +1939,10 @@ function searchMessages(
.run({ conversationId, limit });
}
const prefixedColumns = sqlJoin(
MESSAGE_COLUMNS_FRAGMENTS.map(name => sqlFragment`messages.${name}`)
);
// The `MATCH` is necessary in order to for `snippet()` helper function to
// give us the right results. We can't call `snippet()` in the query above
// because it would bloat the temporary table with text data and we want
@ -1945,9 +1950,7 @@ function searchMessages(
const ftsFragment = sqlFragment`
SELECT
messages.rowid,
messages.json,
messages.sent_at,
messages.received_at,
${prefixedColumns},
snippet(messages_fts, -1, ${SNIPPET_LEFT_PLACEHOLDER}, ${SNIPPET_RIGHT_PLACEHOLDER}, ${SNIPPET_TRUNCATION_PLACEHOLDER}, 10) AS ftsSnippet
FROM tmp_filtered_results
INNER JOIN messages_fts
@ -1966,6 +1969,12 @@ function searchMessages(
const [sqlQuery, params] = sql`${ftsFragment};`;
result = writable.prepare(sqlQuery).all(params);
} else {
const coalescedColumns = MESSAGE_COLUMNS_FRAGMENTS.map(
name => sqlFragment`
COALESCE(messages.${name}, ftsResults.${name}) AS ${name}
`
);
// If contactServiceIdsMatchingQuery is not empty, we due an OUTER JOIN
// between:
// 1) the messages that mention at least one of
@ -1978,9 +1987,7 @@ function searchMessages(
const [sqlQuery, params] = sql`
SELECT
messages.rowid as rowid,
COALESCE(messages.json, ftsResults.json) as json,
COALESCE(messages.sent_at, ftsResults.sent_at) as sent_at,
COALESCE(messages.received_at, ftsResults.received_at) as received_at,
${sqlJoin(coalescedColumns)},
ftsResults.ftsSnippet,
mentionAci,
start as mentionStart,
@ -2082,7 +2089,9 @@ export function getMostRecentAddressableMessages(
limit = 5
): Array<MessageType> {
const [query, parameters] = sql`
SELECT json FROM messages
SELECT
${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
INDEXED BY messages_by_date_addressable
WHERE
conversationId IS ${conversationId} AND
@ -2093,7 +2102,7 @@ export function getMostRecentAddressableMessages(
const rows = db.prepare(query).all(parameters);
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
export function getMostRecentAddressableNondisappearingMessages(
@ -2102,7 +2111,9 @@ export function getMostRecentAddressableNondisappearingMessages(
limit = 5
): Array<MessageType> {
const [query, parameters] = sql`
SELECT json FROM messages
SELECT
${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
INDEXED BY messages_by_date_addressable_nondisappearing
WHERE
expireTimer IS NULL AND
@ -2114,7 +2125,7 @@ export function getMostRecentAddressableNondisappearingMessages(
const rows = db.prepare(query).all(parameters);
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
export function removeSyncTaskById(db: WritableDB, id: string): void {
@ -2248,7 +2259,6 @@ export function saveMessage(
const {
body,
conversationId,
groupV2Change,
hasAttachments,
hasFileAttachments,
hasVisualMediaAttachments,
@ -2257,6 +2267,7 @@ export function saveMessage(
isViewOnce,
mentionsMe,
received_at,
received_at_ms,
schemaVersion,
sent_at,
serverGuid,
@ -2264,13 +2275,22 @@ export function saveMessage(
sourceServiceId,
sourceDevice,
storyId,
timestamp,
type,
readStatus,
expireTimer,
expirationStartTimestamp,
attachments,
seenStatus: originalSeenStatus,
serverTimestamp,
unidentifiedDeliveryReceived,
...json
} = data;
let { seenStatus } = data;
// Extracted separately since we store this field in JSON
const { attachments, groupV2Change } = data;
let seenStatus = originalSeenStatus;
if (attachments) {
strictAssert(
@ -2313,6 +2333,7 @@ export function saveMessage(
isViewOnce: isViewOnce ? 1 : 0,
mentionsMe: mentionsMe ? 1 : 0,
received_at: received_at || null,
received_at_ms: received_at_ms || null,
schemaVersion: schemaVersion || 0,
serverGuid: serverGuid || null,
sent_at: sent_at || null,
@ -2321,43 +2342,22 @@ export function saveMessage(
sourceDevice: sourceDevice || null,
storyId: storyId || null,
type: type || null,
timestamp: timestamp ?? 0,
readStatus: readStatus ?? null,
seenStatus: seenStatus ?? SeenStatus.NotApplicable,
};
serverTimestamp: serverTimestamp ?? null,
unidentifiedDeliveryReceived: unidentifiedDeliveryReceived ? 1 : 0,
} satisfies Omit<MessageTypeUnhydrated, 'json'>;
if (id && !forceSave) {
prepare(
db,
`
UPDATE messages SET
id = $id,
json = $json,
body = $body,
conversationId = $conversationId,
expirationStartTimestamp = $expirationStartTimestamp,
expireTimer = $expireTimer,
hasAttachments = $hasAttachments,
hasFileAttachments = $hasFileAttachments,
hasVisualMediaAttachments = $hasVisualMediaAttachments,
isChangeCreatedByUs = $isChangeCreatedByUs,
isErased = $isErased,
isViewOnce = $isViewOnce,
mentionsMe = $mentionsMe,
received_at = $received_at,
schemaVersion = $schemaVersion,
serverGuid = $serverGuid,
sent_at = $sent_at,
source = $source,
sourceServiceId = $sourceServiceId,
sourceDevice = $sourceDevice,
storyId = $storyId,
type = $type,
readStatus = $readStatus,
seenStatus = $seenStatus
${MESSAGE_COLUMNS.map(name => `${name} = $${name}`).join(', ')}
WHERE id = $id;
`
).run({ ...payloadWithoutJson, json: objectToJSON(data) });
).run({ ...payloadWithoutJson, json: objectToJSON(json) });
if (jobToInsert) {
insertJob(db, jobToInsert);
@ -2366,79 +2366,28 @@ export function saveMessage(
return id;
}
const toCreate = {
...data,
id: id || generateMessageId(data.received_at).id,
};
const createdId = id || generateMessageId(data.received_at).id;
prepare(
db,
`
INSERT INTO messages (
id,
json,
body,
conversationId,
expirationStartTimestamp,
expireTimer,
hasAttachments,
hasFileAttachments,
hasVisualMediaAttachments,
isChangeCreatedByUs,
isErased,
isViewOnce,
mentionsMe,
received_at,
schemaVersion,
serverGuid,
sent_at,
source,
sourceServiceId,
sourceDevice,
storyId,
type,
readStatus,
seenStatus
) values (
$id,
$json,
$body,
$conversationId,
$expirationStartTimestamp,
$expireTimer,
$hasAttachments,
$hasFileAttachments,
$hasVisualMediaAttachments,
$isChangeCreatedByUs,
$isErased,
$isViewOnce,
$mentionsMe,
$received_at,
$schemaVersion,
$serverGuid,
$sent_at,
$source,
$sourceServiceId,
$sourceDevice,
$storyId,
$type,
$readStatus,
$seenStatus
${MESSAGE_COLUMNS.join(', ')}
) VALUES (
${MESSAGE_COLUMNS.map(name => `$${name}`).join(', ')}
);
`
).run({
...payloadWithoutJson,
id: toCreate.id,
json: objectToJSON(toCreate),
id: createdId,
json: objectToJSON(json),
});
if (jobToInsert) {
insertJob(db, jobToInsert);
}
return toCreate.id;
return createdId;
}
function saveMessages(
@ -2507,7 +2456,13 @@ export function getMessageById(
id: string
): MessageType | undefined {
const row = db
.prepare<Query>('SELECT json FROM messages WHERE id = $id;')
.prepare<Query>(
`
SELECT ${MESSAGE_COLUMNS.join(', ')}
FROM messages
WHERE id = $id;
`
)
.get({
id,
});
@ -2516,7 +2471,7 @@ export function getMessageById(
return undefined;
}
return jsonToObject(row.json);
return hydrateMessage(row);
}
function getMessagesById(
@ -2528,22 +2483,30 @@ function getMessagesById(
messageIds,
(batch: ReadonlyArray<string>): Array<MessageType> => {
const query = db.prepare<ArrayQuery>(
`SELECT json FROM messages WHERE id IN (${Array(batch.length)
.fill('?')
.join(',')});`
`
SELECT ${MESSAGE_COLUMNS.join(', ')}
FROM messages
WHERE id IN (
${Array(batch.length).fill('?').join(',')}
);`
);
const rows: JSONRows = query.all(batch);
return rows.map(row => jsonToObject(row.json));
const rows: Array<MessageTypeUnhydrated> = query.all(batch);
return rows.map(row => hydrateMessage(row));
}
);
}
function _getAllMessages(db: ReadableDB): Array<MessageType> {
const rows: JSONRows = db
.prepare<EmptyQuery>('SELECT json FROM messages ORDER BY id ASC;')
const rows: Array<MessageTypeUnhydrated> = db
.prepare<EmptyQuery>(
`
SELECT ${MESSAGE_COLUMNS.join(', ')}
FROM messages ORDER BY id ASC
`
)
.all();
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
function _removeAllMessages(db: WritableDB): void {
db.exec(`
@ -2574,10 +2537,10 @@ function getMessageBySender(
sent_at: number;
}
): MessageType | undefined {
const rows: JSONRows = prepare(
const rows: Array<MessageTypeUnhydrated> = prepare(
db,
`
SELECT json FROM messages WHERE
SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE
(source = $source OR sourceServiceId = $sourceServiceId) AND
sourceDevice = $sourceDevice AND
sent_at = $sent_at
@ -2603,7 +2566,7 @@ function getMessageBySender(
return undefined;
}
return jsonToObject(rows[0].json);
return hydrateMessage(rows[0]);
}
export function _storyIdPredicate(
@ -2667,7 +2630,9 @@ function getUnreadByConversationAndMarkRead(
db.prepare(updateExpirationQuery).run(updateExpirationParams);
const [selectQuery, selectParams] = sql`
SELECT id, json FROM messages
SELECT
${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
WHERE
conversationId = ${conversationId} AND
seenStatus = ${SeenStatus.Unseen} AND
@ -2701,7 +2666,7 @@ function getUnreadByConversationAndMarkRead(
db.prepare(updateStatusQuery).run(updateStatusParams);
return rows.map(row => {
const json = jsonToObject<MessageType>(row.json);
const json = hydrateMessage(row);
return {
originalReadStatus: json.readStatus,
readStatus: ReadStatus.Read,
@ -2929,7 +2894,10 @@ function getRecentStoryReplies(
};
const createQuery = (timeFilter: QueryFragment): QueryFragment => sqlFragment`
SELECT json FROM messages WHERE
SELECT
${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
WHERE
(${messageId} IS NULL OR id IS NOT ${messageId}) AND
isStory IS 0 AND
storyId IS ${storyId} AND
@ -2940,9 +2908,9 @@ function getRecentStoryReplies(
`;
const template = sqlFragment`
SELECT first.json FROM (${createQuery(timeFilters.first)}) as first
SELECT first.* FROM (${createQuery(timeFilters.first)}) as first
UNION ALL
SELECT second.json FROM (${createQuery(timeFilters.second)}) as second
SELECT second.* FROM (${createQuery(timeFilters.second)}) as second
`;
const [query, params] = sql`${template} LIMIT ${limit}`;
@ -2988,7 +2956,9 @@ function getAdjacentMessagesByConversation(
requireFileAttachments;
const createQuery = (timeFilter: QueryFragment): QueryFragment => sqlFragment`
SELECT json FROM messages WHERE
SELECT
${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages WHERE
conversationId = ${conversationId} AND
${
requireDifferentMessage
@ -3014,15 +2984,15 @@ function getAdjacentMessagesByConversation(
`;
let template = sqlFragment`
SELECT first.json FROM (${createQuery(timeFilters.first)}) as first
SELECT first.* FROM (${createQuery(timeFilters.first)}) as first
UNION ALL
SELECT second.json FROM (${createQuery(timeFilters.second)}) as second
SELECT second.* FROM (${createQuery(timeFilters.second)}) as second
`;
// See `filterValidAttachments` in ts/state/ducks/lightbox.ts
if (requireVisualMediaAttachments) {
template = sqlFragment`
SELECT json
SELECT messages.*
FROM (${template}) as messages
WHERE
(
@ -3037,7 +3007,7 @@ function getAdjacentMessagesByConversation(
`;
} else if (requireFileAttachments) {
template = sqlFragment`
SELECT json
SELECT messages.*
FROM (${template}) as messages
WHERE
(
@ -3086,7 +3056,7 @@ function getAllStories(
}
): GetAllStoriesResultType {
const [storiesQuery, storiesParams] = sql`
SELECT json, id
SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
WHERE
isStory = 1 AND
@ -3094,10 +3064,7 @@ function getAllStories(
(${sourceServiceId} IS NULL OR sourceServiceId IS ${sourceServiceId})
ORDER BY received_at ASC, sent_at ASC;
`;
const rows: ReadonlyArray<{
id: string;
json: string;
}> = db.prepare(storiesQuery).all(storiesParams);
const rows = db.prepare(storiesQuery).all(storiesParams);
const [repliesQuery, repliesParams] = sql`
SELECT DISTINCT storyId
@ -3126,7 +3093,7 @@ function getAllStories(
);
return rows.map(row => ({
...jsonToObject(row.json),
...hydrateMessage(row),
hasReplies: Boolean(repliesLookup.has(row.id)),
hasRepliesFromSelf: Boolean(repliesFromSelfLookup.has(row.id)),
}));
@ -3301,7 +3268,7 @@ function getLastConversationActivity(
const row = prepare(
db,
`
SELECT json FROM messages
SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages
INDEXED BY messages_activity
WHERE
conversationId IS $conversationId AND
@ -3320,7 +3287,7 @@ function getLastConversationActivity(
return undefined;
}
return jsonToObject(row.json);
return hydrateMessage(row);
}
function getLastConversationPreview(
db: ReadableDB,
@ -3332,19 +3299,15 @@ function getLastConversationPreview(
includeStoryReplies: boolean;
}
): MessageType | undefined {
type Row = Readonly<{
json: string;
}>;
const index = includeStoryReplies
? 'messages_preview'
: 'messages_preview_without_story';
const row: Row | undefined = prepare(
const row: MessageTypeUnhydrated | undefined = prepare(
db,
`
SELECT json FROM (
SELECT json, expiresAt FROM messages
SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM (
SELECT ${MESSAGE_COLUMNS.join(', ')}, expiresAt FROM messages
INDEXED BY ${index}
WHERE
conversationId IS $conversationId AND
@ -3361,7 +3324,7 @@ function getLastConversationPreview(
now: Date.now(),
});
return row ? jsonToObject(row.json) : undefined;
return row ? hydrateMessage(row) : undefined;
}
function getConversationMessageStats(
@ -3400,7 +3363,7 @@ function getLastConversationMessage(
const row = db
.prepare<Query>(
`
SELECT json FROM messages WHERE
SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages WHERE
conversationId = $conversationId
ORDER BY received_at DESC, sent_at DESC
LIMIT 1;
@ -3414,7 +3377,7 @@ function getLastConversationMessage(
return undefined;
}
return jsonToObject(row.json);
return hydrateMessage(row);
}
function getOldestUnseenMessageForConversation(
@ -3730,7 +3693,7 @@ function getCallHistoryMessageByCallId(
}
): MessageType | undefined {
const [query, params] = sql`
SELECT json
SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
WHERE conversationId = ${options.conversationId}
AND type = 'call-history'
@ -3740,7 +3703,7 @@ function getCallHistoryMessageByCallId(
if (row == null) {
return;
}
return jsonToObject(row.json);
return hydrateMessage(row);
}
function getCallHistory(
@ -4524,45 +4487,57 @@ function getMessagesBySentAt(
db: ReadableDB,
sentAt: number
): Array<MessageType> {
// Make sure to preserve order of columns
const editedColumns = MESSAGE_COLUMNS_FRAGMENTS.map(name => {
if (name.fragment === 'received_at' || name.fragment === 'sent_at') {
return name;
}
return sqlFragment`messages.${name}`;
});
const [query, params] = sql`
SELECT messages.json, received_at, sent_at FROM edited_messages
SELECT ${sqlJoin(editedColumns)}
FROM edited_messages
INNER JOIN messages ON
messages.id = edited_messages.messageId
WHERE edited_messages.sentAt = ${sentAt}
UNION
SELECT json, received_at, sent_at FROM messages
SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}
FROM messages
WHERE sent_at = ${sentAt}
ORDER BY messages.received_at DESC, messages.sent_at DESC;
`;
const rows = db.prepare(query).all(params);
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
function getExpiredMessages(db: ReadableDB): Array<MessageType> {
const now = Date.now();
const rows: JSONRows = db
const rows: Array<MessageTypeUnhydrated> = db
.prepare<Query>(
`
SELECT json FROM messages WHERE
SELECT ${sqlJoin(MESSAGE_COLUMNS_FRAGMENTS)}, expiresAt
FROM messages
WHERE
expiresAt <= $now
ORDER BY expiresAt ASC;
`
)
.all({ now });
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
function getMessagesUnexpectedlyMissingExpirationStartTimestamp(
db: ReadableDB
): Array<MessageType> {
const rows: JSONRows = db
const rows: Array<MessageTypeUnhydrated> = db
.prepare<EmptyQuery>(
`
SELECT json FROM messages
SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages
INDEXED BY messages_unexpectedly_missing_expiration_start_timestamp
WHERE
expireTimer > 0 AND
@ -4579,7 +4554,7 @@ function getMessagesUnexpectedlyMissingExpirationStartTimestamp(
)
.all();
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
function getSoonestMessageExpiry(db: ReadableDB): undefined | number {
@ -4607,7 +4582,7 @@ function getNextTapToViewMessageTimestampToAgeOut(
const row = db
.prepare<EmptyQuery>(
`
SELECT json FROM messages
SELECT ${MESSAGE_COLUMNS.join(', ')} FROM messages
WHERE
-- we want this query to use the messages_view_once index rather than received_at
likelihood(isViewOnce = 1, 0.01)
@ -4621,7 +4596,7 @@ function getNextTapToViewMessageTimestampToAgeOut(
if (!row) {
return undefined;
}
const data = jsonToObject<MessageType>(row.json);
const data = hydrateMessage(row);
const result = data.received_at_ms;
return isNormalNumber(result) ? result : undefined;
}
@ -4630,16 +4605,16 @@ function getTapToViewMessagesNeedingErase(
db: ReadableDB,
maxTimestamp: number
): Array<MessageType> {
const rows: JSONRows = db
const rows: Array<MessageTypeUnhydrated> = db
.prepare<Query>(
`
SELECT json
SELECT ${MESSAGE_COLUMNS.join(', ')}
FROM messages
WHERE
isViewOnce = 1
AND (isErased IS NULL OR isErased != 1)
AND (
IFNULL(json ->> '$.received_at_ms', 0) <= $maxTimestamp
IFNULL(received_at_ms, 0) <= $maxTimestamp
)
`
)
@ -4647,7 +4622,7 @@ function getTapToViewMessagesNeedingErase(
maxTimestamp,
});
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
const MAX_UNPROCESSED_ATTEMPTS = 10;
@ -6716,10 +6691,10 @@ function getMessagesNeedingUpgrade(
limit: number,
{ maxVersion }: { maxVersion: number }
): Array<MessageType> {
const rows: JSONRows = db
const rows: Array<MessageTypeUnhydrated> = db
.prepare<Query>(
`
SELECT json
SELECT ${MESSAGE_COLUMNS.join(', ')}
FROM messages
WHERE
(schemaVersion IS NULL OR schemaVersion < $maxVersion) AND
@ -6736,7 +6711,7 @@ function getMessagesNeedingUpgrade(
limit,
});
return rows.map(row => jsonToObject(row.json));
return rows.map(row => hydrateMessage(row));
}
// Exported for tests
@ -7023,12 +6998,14 @@ function pageMessages(
rowids,
(batch: ReadonlyArray<number>): Array<MessageType> => {
const query = writable.prepare<ArrayQuery>(
`SELECT json FROM messages WHERE rowid IN (${Array(batch.length)
.fill('?')
.join(',')});`
`
SELECT ${MESSAGE_COLUMNS.join(', ')}
FROM messages
WHERE rowid IN (${Array(batch.length).fill('?').join(',')});
`
);
const rows: JSONRows = query.all(batch);
return rows.map(row => jsonToObject(row.json));
const rows: Array<MessageTypeUnhydrated> = query.all(batch);
return rows.map(row => hydrateMessage(row));
}
);
@ -7446,11 +7423,14 @@ function getUnreadEditedMessagesAndMarkRead(
}
): GetUnreadByConversationAndMarkReadResultType {
return db.transaction(() => {
const editedColumns = MESSAGE_COLUMNS_FRAGMENTS.filter(
name => name.fragment !== 'sent_at' && name.fragment !== 'readStatus'
).map(name => sqlFragment`messages.${name}`);
const [selectQuery, selectParams] = sql`
SELECT
messages.id,
messages.json,
edited_messages.sentAt,
${sqlJoin(editedColumns)},
edited_messages.sentAt as sent_at,
edited_messages.readStatus
FROM edited_messages
JOIN messages
@ -7481,7 +7461,7 @@ function getUnreadEditedMessagesAndMarkRead(
}
return rows.map(row => {
const json = jsonToObject<MessageType>(row.json);
const json = hydrateMessage(row);
return {
originalReadStatus: row.readStatus,
readStatus: ReadStatus.Read,
@ -7494,8 +7474,6 @@ function getUnreadEditedMessagesAndMarkRead(
'sourceServiceId',
'type',
]),
// Use the edited message timestamp
sent_at: row.sentAt,
};
});
})();

88
ts/sql/hydration.ts Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { ReadStatus } from '../messages/MessageReadStatus';
import type { SeenStatus } from '../MessageSeenStatus';
import type { ServiceIdString } from '../types/ServiceId';
import { dropNull } from '../util/dropNull';
/* eslint-disable camelcase */
import type {
MessageTypeUnhydrated,
MessageType,
MESSAGE_COLUMNS,
} from './Interface';
function toBoolean(value: number | null): boolean | undefined {
if (value == null) {
return undefined;
}
return value === 1;
}
export function hydrateMessage(row: MessageTypeUnhydrated): MessageType {
const {
json,
id,
body,
conversationId,
expirationStartTimestamp,
expireTimer,
hasAttachments,
hasFileAttachments,
hasVisualMediaAttachments,
isErased,
isViewOnce,
mentionsMe,
received_at,
received_at_ms,
schemaVersion,
serverGuid,
sent_at,
source,
sourceServiceId,
sourceDevice,
storyId,
type,
readStatus,
seenStatus,
timestamp,
serverTimestamp,
unidentifiedDeliveryReceived,
} = row;
return {
...(JSON.parse(json) as Omit<
MessageType,
(typeof MESSAGE_COLUMNS)[number]
>),
id,
body: dropNull(body),
conversationId: conversationId || '',
expirationStartTimestamp: dropNull(expirationStartTimestamp),
expireTimer: dropNull(expireTimer) as MessageType['expireTimer'],
hasAttachments: toBoolean(hasAttachments),
hasFileAttachments: toBoolean(hasFileAttachments),
hasVisualMediaAttachments: toBoolean(hasVisualMediaAttachments),
isErased: toBoolean(isErased),
isViewOnce: toBoolean(isViewOnce),
mentionsMe: toBoolean(mentionsMe),
received_at: received_at || 0,
received_at_ms: dropNull(received_at_ms),
schemaVersion: dropNull(schemaVersion),
serverGuid: dropNull(serverGuid),
sent_at: sent_at || 0,
source: dropNull(source),
sourceServiceId: dropNull(sourceServiceId) as ServiceIdString | undefined,
sourceDevice: dropNull(sourceDevice),
storyId: dropNull(storyId),
type: type as MessageType['type'],
readStatus: readStatus == null ? undefined : (readStatus as ReadStatus),
seenStatus: seenStatus == null ? undefined : (seenStatus as SeenStatus),
timestamp: timestamp || 0,
serverTimestamp: dropNull(serverTimestamp),
unidentifiedDeliveryReceived: toBoolean(unidentifiedDeliveryReceived),
};
}

53
ts/sql/migrations/1270-normalize-messages.ts Normal file
View file

@ -0,0 +1,53 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Database } from '@signalapp/better-sqlite3';
import type { LoggerType } from '../../types/Logging';
import { sql } from '../util';
export const version = 1270;
export function updateToSchemaVersion1270(
currentVersion: number,
db: Database,
logger: LoggerType
): void {
if (currentVersion >= 1270) {
return;
}
db.transaction(() => {
const [query] = sql`
ALTER TABLE messages
ADD COLUMN timestamp INTEGER;
ALTER TABLE messages
ADD COLUMN received_at_ms INTEGER;
ALTER TABLE messages
ADD COLUMN unidentifiedDeliveryReceived INTEGER;
ALTER TABLE messages
ADD COLUMN serverTimestamp INTEGER;
ALTER TABLE messages
RENAME COLUMN source TO legacySource;
ALTER TABLE messages
ADD COLUMN source TEXT;
UPDATE messages SET
timestamp = json_extract(json, '$.timestamp'),
received_at_ms = json_extract(json, '$.received_at_ms'),
unidentifiedDeliveryReceived =
json_extract(json, '$.unidentifiedDeliveryReceived'),
serverTimestamp =
json_extract(json, '$.serverTimestamp'),
source = IFNULL(json_extract(json, '$.source'), '+' || legacySource);
ALTER TABLE messages
DROP COLUMN legacySource;
`;
db.exec(query);
db.pragma('user_version = 1270');
})();
logger.info('updateToSchemaVersion1270: success!');
}

8
ts/sql/migrations/51-centralize-conversation-jobs.ts
View file

@ -4,7 +4,7 @@
import type { LoggerType } from '../../types/Logging';
import { isRecord } from '../../util/isRecord';
import type { WritableDB } from '../Interface';
import { getJobsInQueue, getMessageById, insertJob } from '../Server';
import { getJobsInQueue, insertJob } from '../Server';
export default function updateToSchemaVersion51(
currentVersion: number,
@ -24,6 +24,10 @@ export default function updateToSchemaVersion51(
const reactionsJobs = getJobsInQueue(db, 'reactions');
deleteJobsInQueue.run({ queueType: 'reactions' });
const getMessageById = db.prepare(
'SELECT conversationId FROM messages WHERE id IS ?'
);
reactionsJobs.forEach(job => {
const { data, id } = job;
@ -42,7 +46,7 @@ export default function updateToSchemaVersion51(
return;
}
const message = getMessageById(db, messageId);
const message = getMessageById.get(messageId);
if (!message) {
logger.warn(
`updateToSchemaVersion51: Unable to find message for reaction job ${id}`

8
ts/sql/migrations/78-merge-receipt-jobs.ts
View file

@ -4,7 +4,7 @@
import type { LoggerType } from '../../types/Logging';
import { isRecord } from '../../util/isRecord';
import type { WritableDB } from '../Interface';
import { getJobsInQueue, getMessageById, insertJob } from '../Server';
import { getJobsInQueue, insertJob } from '../Server';
export default function updateToSchemaVersion78(
currentVersion: number,
@ -41,6 +41,10 @@ export default function updateToSchemaVersion78(
},
];
const getMessageById = db.prepare(
'SELECT conversationId FROM messages WHERE id IS ?'
);
for (const queue of queues) {
const prevJobs = getJobsInQueue(db, queue.queueType);
deleteJobsInQueue.run({ queueType: queue.queueType });
@ -62,7 +66,7 @@ export default function updateToSchemaVersion78(
return;
}
const message = getMessageById(db, messageId);
const message = getMessageById.get(messageId);
if (!message) {
logger.warn(
`updateToSchemaVersion78: Unable to find message for ${queue.queueType} job ${id}`

6
ts/sql/migrations/index.ts
View file

@ -102,10 +102,11 @@ import { updateToSchemaVersion1220 } from './1220-blob-sessions';
import { updateToSchemaVersion1230 } from './1230-call-links-admin-key-index';
import { updateToSchemaVersion1240 } from './1240-defunct-call-links-table';
import { updateToSchemaVersion1250 } from './1250-defunct-call-links-storage';
import { updateToSchemaVersion1260 } from './1260-sync-tasks-rowid';
import {
updateToSchemaVersion1260,
updateToSchemaVersion1270,
version as MAX_VERSION,
} from './1260-sync-tasks-rowid';
} from './1270-normalize-messages';
import { DataWriter } from '../Server';
function updateToSchemaVersion1(
@ -2078,6 +2079,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion1240,
updateToSchemaVersion1250,
updateToSchemaVersion1260,
updateToSchemaVersion1270,
];
export class DBVersionFromFutureError extends Error {

3
ts/sql/util.ts
View file

@ -11,7 +11,8 @@ export type ArrayQuery = Array<ReadonlyArray<null | number | bigint | string>>;
export type Query = {
[key: string]: null | number | bigint | string | Uint8Array;
};
export type JSONRows = Array<{ readonly json: string }>;
export type JSONRow = Readonly<{ json: string }>;
export type JSONRows = Array<JSONRow>;
export type TableType =
| 'attachment_downloads'

18
ts/test-electron/backup/attachments_test.ts
View file

@ -166,8 +166,8 @@ describe('backup/attachments', () => {
// path & iv will not be roundtripped
[
composeMessage(1, {
hasAttachments: 1,
hasVisualMediaAttachments: 1,
hasAttachments: true,
hasVisualMediaAttachments: true,
attachments: [
omit(longMessageAttachment, NON_ROUNDTRIPPED_FIELDS),
omit(normalAttachment, NON_ROUNDTRIPPED_FIELDS),
@ -284,8 +284,8 @@ describe('backup/attachments', () => {
// path & iv will not be roundtripped
[
composeMessage(1, {
hasAttachments: 1,
hasVisualMediaAttachments: 1,
hasAttachments: true,
hasVisualMediaAttachments: true,
attachments: [
omit(attachment1, NON_ROUNDTRIPPED_FIELDS),
omit(attachment2, NON_ROUNDTRIPPED_FIELDS),
@ -307,8 +307,8 @@ describe('backup/attachments', () => {
],
[
composeMessage(1, {
hasAttachments: 1,
hasVisualMediaAttachments: 1,
hasAttachments: true,
hasVisualMediaAttachments: true,
// path, iv, and uploadTimestamp will not be roundtripped,
// but there will be a backupLocator
@ -341,7 +341,7 @@ describe('backup/attachments', () => {
],
[
composeMessage(1, {
hasAttachments: 1,
hasAttachments: true,
attachments: [
{
...omit(attachment, NON_ROUNDTRIPPED_BACKUP_LOCATOR_FIELDS),
@ -604,8 +604,8 @@ describe('backup/attachments', () => {
[
{
...existingMessage,
hasAttachments: 1,
hasVisualMediaAttachments: 1,
hasAttachments: true,
hasVisualMediaAttachments: true,
attachments: [
{
...omit(

16
ts/test-electron/backup/helpers.ts
View file

@ -110,10 +110,20 @@ function sortAndNormalize(
// Get rid of unserializable `undefined` values.
return JSON.parse(
JSON.stringify({
// Migration defaults
hasAttachments: 0,
// Defaults
hasAttachments: false,
hasFileAttachments: false,
hasVisualMediaAttachments: false,
isErased: false,
isViewOnce: false,
mentionsMe: false,
seenStatus: 0,
readStatus: 0,
unidentifiedDeliveryReceived: false,
// Drop more `undefined` values
...JSON.parse(JSON.stringify(rest)),
...rest,
conversationId: mapConvoId(conversationId),
reactions: reactions?.map(({ fromId, ...restOfReaction }) => {
return {

1
ts/test-electron/backup/non_bubble_test.ts
View file

@ -76,7 +76,6 @@ describe('backup/non-bubble messages', () => {
flags: Proto.DataMessage.Flags.END_SESSION,
attachments: [],
contact: [],
hasAttachments: 0,
},
]);
});

25
ts/test-node/sql/migration_1060_test.ts
View file

@ -6,11 +6,11 @@ import { v4 as generateGuid } from 'uuid';
import {
dequeueOldestSyncTasks,
getMostRecentAddressableMessages,
removeSyncTaskById,
saveSyncTasks,
} from '../../sql/Server';
import type { WritableDB } from '../../sql/Interface';
import type { WritableDB, ReadableDB, MessageType } from '../../sql/Interface';
import { sql, jsonToObject } from '../../sql/util';
import { insertData, updateToVersion, createDB } from './helpers';
import { MAX_SYNC_TASK_ATTEMPTS } from '../../util/syncTasks.types';
import { WEEK } from '../../util/durations';
@ -20,6 +20,27 @@ import type { SyncTaskType } from '../../util/syncTasks';
/* eslint-disable camelcase */
// Snapshot before: 1270
export function getMostRecentAddressableMessages(
db: ReadableDB,
conversationId: string,
limit = 5
): Array<MessageType> {
const [query, parameters] = sql`
SELECT json FROM messages
INDEXED BY messages_by_date_addressable
WHERE
conversationId IS ${conversationId} AND
isAddressableMessage = 1
ORDER BY received_at DESC, sent_at DESC
LIMIT ${limit};
`;
const rows = db.prepare(query).all(parameters);
return rows.map(row => jsonToObject(row.json));
}
function generateMessage(json: MessageAttributesType) {
const { conversationId, received_at, sent_at, type } = json;

26
ts/test-node/sql/migration_1080_test.ts
View file

@ -4,8 +4,8 @@
import { assert } from 'chai';
import { v4 as generateGuid } from 'uuid';
import type { WritableDB } from '../../sql/Interface';
import { getMostRecentAddressableNondisappearingMessages } from '../../sql/Server';
import type { WritableDB, ReadableDB, MessageType } from '../../sql/Interface';
import { sql, jsonToObject } from '../../sql/util';
import { createDB, insertData, updateToVersion } from './helpers';
import type { MessageAttributesType } from '../../model-types';
@ -26,6 +26,28 @@ function generateMessage(json: MessageAttributesType) {
};
}
// Snapshot before: 1270
export function getMostRecentAddressableNondisappearingMessages(
db: ReadableDB,
conversationId: string,
limit = 5
): Array<MessageType> {
const [query, parameters] = sql`
SELECT json FROM messages
INDEXED BY messages_by_date_addressable_nondisappearing
WHERE
expireTimer IS NULL AND
conversationId IS ${conversationId} AND
isAddressableMessage = 1
ORDER BY received_at DESC, sent_at DESC
LIMIT ${limit};
`;
const rows = db.prepare(query).all(parameters);
return rows.map(row => jsonToObject(row.json));
}
describe('SQL/updateToSchemaVersion1080', () => {
let db: WritableDB;
beforeEach(() => {

12
ts/test-node/sql/migrations_test.ts
View file

@ -1351,10 +1351,8 @@ describe('SQL migrations test', () => {
db.exec(
`
INSERT INTO messages
(id, json)
VALUES ('${MESSAGE_ID_1}', '${JSON.stringify({
conversationId: CONVERSATION_ID_1,
})}')
(id, conversationId)
VALUES ('${MESSAGE_ID_1}', '${CONVERSATION_ID_1}');
`
);
@ -2482,10 +2480,8 @@ describe('SQL migrations test', () => {
db.exec(
`
INSERT INTO messages
(id, json)
VALUES ('${MESSAGE_ID_1}', '${JSON.stringify({
conversationId: CONVERSATION_ID_1,
})}')
(id, conversationId)
VALUES ('${MESSAGE_ID_1}', '${CONVERSATION_ID_1}');
`
);

5
ts/test-node/types/Message2_test.ts
View file

@ -126,11 +126,9 @@ describe('Message', () => {
it('should initialize schema version to zero', () => {
const input = getDefaultMessage({
body: 'Imagine there is no heaven…',
attachments: [],
});
const expected = getDefaultMessage({
body: 'Imagine there is no heaven…',
attachments: [],
schemaVersion: 0,
});
@ -203,7 +201,6 @@ describe('Message', () => {
hasVisualMediaAttachments: undefined,
hasFileAttachments: undefined,
schemaVersion: Message.CURRENT_SCHEMA_VERSION,
contact: [],
});
const expectedAttachmentData = 'It’s easy if you try';
@ -655,7 +652,6 @@ describe('Message', () => {
});
const expected = getDefaultMessage({
body: 'hey there!',
contact: [],
});
const result = await upgradeVersion(message, getDefaultContext());
assert.deepEqual(result, expected);
@ -848,7 +844,6 @@ describe('Message', () => {
key: 'key',
},
],
contact: [],
});
const result = await Message.upgradeSchema(message, {
...getDefaultContext(),

42
ts/types/Message2.ts
View file

@ -285,6 +285,10 @@ export const _mapAttachments =
message: MessageAttributesType,
context: ContextType
): Promise<MessageAttributesType> => {
if (!message.attachments?.length) {
return message;
}
const upgradeWithContext = esbuildAnonymize((attachment: AttachmentType) =>
upgradeAttachment(attachment, context, message)
);
@ -356,6 +360,10 @@ export const _mapContact =
message: MessageAttributesType,
context: ContextType
): Promise<MessageAttributesType> => {
if (!message.contact?.length) {
return message;
}
const upgradeWithContext = esbuildAnonymize(
(contact: EmbeddedContactType) =>
upgradeContact(contact, context, message)
@ -501,23 +509,25 @@ const toVersion10 = _withSchemaVersion({
schemaVersion: 10,
upgrade: async (message, context) => {
const processPreviews = _mapPreviewAttachments(migrateDataToFileSystem);
const processSticker = async (
stickerMessage: MessageAttributesType,
stickerContext: ContextType
): Promise<MessageAttributesType> => {
const { sticker } = stickerMessage;
if (!sticker || !sticker.data || !sticker.data.data) {
return stickerMessage;
}
const processSticker = esbuildAnonymize(
async (
stickerMessage: MessageAttributesType,
stickerContext: ContextType
): Promise<MessageAttributesType> => {
const { sticker } = stickerMessage;
if (!sticker || !sticker.data || !sticker.data.data) {
return stickerMessage;
}
return {
...stickerMessage,
sticker: {
...sticker,
data: await migrateDataToFileSystem(sticker.data, stickerContext),
},
};
};
return {
...stickerMessage,
sticker: {
...sticker,
data: await migrateDataToFileSystem(sticker.data, stickerContext),
},
};
}
);
const previewProcessed = await processPreviews(message, context);
const stickerProcessed = await processSticker(previewProcessed, context);