From 60d7cbff3e3e0ea716078dae348becdf372cb030 Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:17:03 -0700 Subject: [PATCH] Migrate message ids to UUIDv7 --- ACKNOWLEDGMENTS.md | 20 ++----- package-lock.json | 28 +++++----- package.json | 4 +- ts/background.ts | 11 ++-- ts/components/EditNicknameAndNoteModal.tsx | 2 +- ts/components/ListTile.tsx | 2 +- ts/components/Preferences.tsx | 2 +- ts/components/SafetyTipsModal.tsx | 2 +- .../ConversationNotificationsModal.tsx | 2 +- ts/groups.ts | 35 +++--------- ts/jobs/JobQueue.ts | 2 +- ts/jobs/helpers/sendReaction.ts | 5 +- ts/models/conversations.ts | 38 +++++-------- ts/models/messages.ts | 5 +- ts/reactions/enqueueReactionForSend.ts | 54 ++++++++++--------- ts/services/backups/import.ts | 26 +++++++-- ts/sql/Client.ts | 4 +- ts/sql/Server.ts | 34 +++++++----- ts/test-electron/MessageReceipts_test.ts | 2 +- ts/test-node/util/callDisposition_test.ts | 2 +- ts/util/generateMessageId.ts | 47 ++++++++++++++++ ts/util/incrementMessageCounter.ts | 5 +- ts/util/isValidUuid.ts | 16 ++++++ ts/windows/attachments.ts | 2 +- 24 files changed, 203 insertions(+), 147 deletions(-) create mode 100644 ts/util/generateMessageId.ts diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 9523cc9ab845..99a8cf01b4a3 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -3352,25 +3352,13 @@ Signal Desktop makes use of the following open source projects. The MIT License (MIT) - Copyright (c) 2010-2016 Robert Kieffer and other contributors + Copyright (c) 2010-2020 Robert Kieffer and other contributors - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ## uuid-browser diff --git a/package-lock.json b/package-lock.json index 280c46725015..d1352c1a8896 100644 --- a/package-lock.json +++ b/package-lock.json @@ -105,7 +105,7 @@ "split2": "4.0.0", "type-fest": "4.23.0", "urlpattern-polyfill": "9.0.0", - "uuid": "3.3.2", + "uuid": "10.0.0", "uuid-browser": "3.1.0", "websocket": "1.0.34", "write-file-atomic": "5.0.1", @@ -184,7 +184,7 @@ "@types/split2": "3.2.1", "@types/terser-webpack-plugin": "5.0.3", "@types/unzipper": "0.10.9", - "@types/uuid": "3.4.4", + "@types/uuid": "10.0.0", "@types/websocket": "1.0.0", "@types/write-file-atomic": "4.0.3", "@types/yargs": "17.0.7", @@ -11446,13 +11446,11 @@ } }, "node_modules/@types/uuid": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.4.tgz", - "integrity": "sha512-tPIgT0GUmdJQNSHxp0X2jnpQfBSTfGxUMc/2CXBU2mnyTFVYVa2ojpoQ74w0U2yn2vw3jnC640+77lkFFpdVDw==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", "dev": true, - "dependencies": { - "@types/node": "*" - } + "license": "MIT" }, "node_modules/@types/wait-on": { "version": "5.3.4", @@ -35778,12 +35776,16 @@ } }, "node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", "bin": { - "uuid": "bin/uuid" + "uuid": "dist/bin/uuid" } }, "node_modules/uuid-browser": { diff --git a/package.json b/package.json index 0d9b42ac30ce..7152843558bc 100644 --- a/package.json +++ b/package.json @@ -189,7 +189,7 @@ "split2": "4.0.0", "type-fest": "4.23.0", "urlpattern-polyfill": "9.0.0", - "uuid": "3.3.2", + "uuid": "10.0.0", "uuid-browser": "3.1.0", "websocket": "1.0.34", "write-file-atomic": "5.0.1", @@ -268,7 +268,7 @@ "@types/split2": "3.2.1", "@types/terser-webpack-plugin": "5.0.3", "@types/unzipper": "0.10.9", - "@types/uuid": "3.4.4", + "@types/uuid": "10.0.0", "@types/websocket": "1.0.0", "@types/write-file-atomic": "4.0.3", "@types/yargs": "17.0.7", diff --git a/ts/background.ts b/ts/background.ts index 1af60dc3004b..7b36cc129b68 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -6,7 +6,7 @@ import { render } from 'react-dom'; import { batch as batchDispatch } from 'react-redux'; import PQueue from 'p-queue'; import pMap from 'p-map'; -import { v4 as generateUuid } from 'uuid'; +import { v7 as generateUuid } from 'uuid'; import * as Registration from './util/registration'; import MessageReceiver from './textsecure/MessageReceiver'; @@ -169,6 +169,7 @@ import { incrementMessageCounter, initializeMessageCounter, } from './util/incrementMessageCounter'; +import { generateMessageId } from './util/generateMessageId'; import { RetryPlaceholders } from './util/retryPlaceholders'; import { setBatchingStrategy } from './util/messageBatcher'; import { parseRemoteClientExpiration } from './util/parseRemoteClientExpiration'; @@ -2694,7 +2695,8 @@ export async function startApp(): Promise { } const partialMessage: MessageAttributesType = { - id: generateUuid(), + ...generateMessageId(data.receivedAtCounter), + canReplyToStory: data.message.isStory ? data.message.canReplyToStory : undefined, @@ -2705,7 +2707,6 @@ export async function startApp(): Promise { ), readStatus: ReadStatus.Read, received_at_ms: data.receivedAtDate, - received_at: data.receivedAtCounter, seenStatus: SeenStatus.NotApplicable, sendStateByConversationId, sent_at: timestamp, @@ -2957,13 +2958,13 @@ export async function startApp(): Promise { `Did not receive receivedAtCounter for message: ${data.timestamp}` ); const partialMessage: MessageAttributesType = { - id: generateUuid(), + ...generateMessageId(data.receivedAtCounter), + canReplyToStory: data.message.isStory ? data.message.canReplyToStory : undefined, conversationId: descriptor.id, readStatus: ReadStatus.Unread, - received_at: data.receivedAtCounter, received_at_ms: data.receivedAtDate, seenStatus: SeenStatus.Unseen, sent_at: data.timestamp, diff --git a/ts/components/EditNicknameAndNoteModal.tsx b/ts/components/EditNicknameAndNoteModal.tsx index 59b323c9b174..898faa447c5e 100644 --- a/ts/components/EditNicknameAndNoteModal.tsx +++ b/ts/components/EditNicknameAndNoteModal.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import type { FormEvent } from 'react'; import React, { useCallback, useMemo, useState } from 'react'; -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import { z } from 'zod'; import { Modal } from './Modal'; import type { LocalizerType } from '../types/I18N'; diff --git a/ts/components/ListTile.tsx b/ts/components/ListTile.tsx index 6ad78d6f2dae..e8fbf155c78c 100644 --- a/ts/components/ListTile.tsx +++ b/ts/components/ListTile.tsx @@ -3,7 +3,7 @@ import classNames from 'classnames'; import React, { useMemo } from 'react'; -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import { getClassNamesFor } from '../util/getClassNamesFor'; import { CircleCheckbox } from './CircleCheckbox'; diff --git a/ts/components/Preferences.tsx b/ts/components/Preferences.tsx index 6a6cd3cd62c7..7bca5ed8db0a 100644 --- a/ts/components/Preferences.tsx +++ b/ts/components/Preferences.tsx @@ -12,7 +12,7 @@ import React, { } from 'react'; import { noop, partition } from 'lodash'; import classNames from 'classnames'; -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import * as LocaleMatcher from '@formatjs/intl-localematcher'; import type { MediaDeviceSettings } from '../types/Calling'; diff --git a/ts/components/SafetyTipsModal.tsx b/ts/components/SafetyTipsModal.tsx index cc8294d3fa3f..0f798db5a52d 100644 --- a/ts/components/SafetyTipsModal.tsx +++ b/ts/components/SafetyTipsModal.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import type { UIEvent } from 'react'; import React, { useEffect, useMemo, useRef, useState } from 'react'; -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import type { LocalizerType } from '../types/I18N'; import { Modal } from './Modal'; import { Button, ButtonVariant } from './Button'; diff --git a/ts/components/conversation/conversation-details/ConversationNotificationsModal.tsx b/ts/components/conversation/conversation-details/ConversationNotificationsModal.tsx index f52d0a6f2242..6d0b1fdbc71a 100644 --- a/ts/components/conversation/conversation-details/ConversationNotificationsModal.tsx +++ b/ts/components/conversation/conversation-details/ConversationNotificationsModal.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import React, { useMemo, useState } from 'react'; -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import type { LocalizerType } from '../../../types/Util'; import { getMuteOptions } from '../../../util/getMuteOptions'; diff --git a/ts/groups.ts b/ts/groups.ts index c46622b66a08..d56033da916d 100644 --- a/ts/groups.ts +++ b/ts/groups.ts @@ -12,7 +12,6 @@ import { } from 'lodash'; import Long from 'long'; import type { ClientZkGroupCipher } from '@signalapp/libsignal-client/zkgroup'; -import { v4 as getGuid } from 'uuid'; import LRU from 'lru-cache'; import * as log from './logging/log'; import { @@ -101,6 +100,7 @@ import { decodeGroupSendEndorsementResponse, isValidGroupSendEndorsementsExpiration, } from './util/groupSendEndorsements'; +import { generateMessageId } from './util/generateMessageId'; type AccessRequiredEnum = Proto.AccessControl.AccessRequired; @@ -293,7 +293,7 @@ type UploadedAvatarType = { type BasicMessageType = Pick< MessageAttributesType, - 'id' | 'schemaVersion' | 'readStatus' | 'seenStatus' + 'readStatus' | 'seenStatus' >; type GroupV2ChangeMessageType = { @@ -335,14 +335,6 @@ const SUPPORTED_CHANGE_EPOCH = 5; export const LINK_VERSION_ERROR = 'LINK_VERSION_ERROR'; const GROUP_INVITE_LINK_PASSWORD_LENGTH = 16; -function generateBasicMessage(): BasicMessageType { - return { - id: getGuid(), - schemaVersion: MAX_MESSAGE_SCHEMA, - // this is missing most properties to fulfill this type - }; -} - // Group Links export function generateGroupInviteLinkPassword(): Uint8Array { @@ -2024,12 +2016,13 @@ export async function createGroupV2( }); const createdTheGroupMessage: MessageAttributesType = { - ...generateBasicMessage(), + ...generateMessageId(incrementMessageCounter()), + + schemaVersion: MAX_MESSAGE_SCHEMA, type: 'group-v2-change', sourceServiceId: ourAci, conversationId: conversation.id, readStatus: ReadStatus.Read, - received_at: incrementMessageCounter(), received_at_ms: timestamp, timestamp, seenStatus: SeenStatus.Seen, @@ -2468,7 +2461,6 @@ export async function initiateMigrationToGroupV2( const groupChangeMessages: Array = []; groupChangeMessages.push({ - ...generateBasicMessage(), type: 'group-v1-migration', groupMigration: { areWeInvited: false, @@ -2609,7 +2601,6 @@ export function buildMigrationBubble( ); return { - ...generateBasicMessage(), type: 'group-v1-migration', groupMigration: { areWeInvited, @@ -2623,7 +2614,6 @@ export function buildMigrationBubble( export function getBasicMigrationBubble(): GroupChangeMessageType { return { - ...generateBasicMessage(), type: 'group-v1-migration', groupMigration: { areWeInvited: false, @@ -2697,7 +2687,6 @@ export async function joinGroupV2ViaLinkAndMigrate({ }; const groupChangeMessages: Array = [ { - ...generateBasicMessage(), type: 'group-v1-migration', groupMigration: { areWeInvited: false, @@ -2865,7 +2854,6 @@ export async function respondToGroupV2Migration({ seenStatus: SeenStatus.Seen, }, { - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { details: [ @@ -2946,7 +2934,6 @@ export async function respondToGroupV2Migration({ if (!areWeInvited && !areWeMember) { // Add a message to the timeline saying the user was removed. This shouldn't happen. groupChangeMessages.push({ - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { details: [ @@ -3184,8 +3171,9 @@ async function updateGroup( return { ...changeMessage, + ...generateMessageId(finalReceivedAt), + schemaVersion: MAX_MESSAGE_SCHEMA, conversationId: conversation.id, - received_at: finalReceivedAt, received_at_ms: syntheticSentAt, sent_at: syntheticSentAt, timestamp, @@ -3895,7 +3883,6 @@ async function updateGroupViaSingleChange({ catchupMessages.length > 0 ) { groupChangeMessages.push({ - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { details: [ @@ -4901,7 +4888,6 @@ function extractDiffs({ if (firstUpdate && serviceIdKindInvitedToGroup !== undefined) { // Note, we will add 'you were invited' to group even if dropInitialJoinMessage = true message = { - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { from: whoInvitedUsUserId || from, @@ -4919,7 +4905,6 @@ function extractDiffs({ }; } else if (firstUpdate && areWePendingApproval) { message = { - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { from: ourAci, @@ -4940,7 +4925,6 @@ function extractDiffs({ sourceServiceId === ourAci ) { message = { - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { from, @@ -4962,7 +4946,6 @@ function extractDiffs({ ); message = { - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { from, @@ -4973,7 +4956,6 @@ function extractDiffs({ }; } else if (firstUpdate && current.revision === 0) { message = { - ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { from, @@ -5002,7 +4984,6 @@ function extractDiffs({ } message = { - ...generateBasicMessage(), type: 'group-v2-change', sourceServiceId, groupV2Change: { @@ -5014,7 +4995,6 @@ function extractDiffs({ }; } else if (details.length > 0) { message = { - ...generateBasicMessage(), type: 'group-v2-change', sourceServiceId, groupV2Change: { @@ -5041,7 +5021,6 @@ function extractDiffs({ `extractDiffs/${logId}: generating change notification for new ${expireTimer} timer` ); timerNotification = { - ...generateBasicMessage(), type: 'timer-notification', sourceServiceId: isReJoin ? undefined : sourceServiceId, flags: Proto.DataMessage.Flags.EXPIRATION_TIMER_UPDATE, diff --git a/ts/jobs/JobQueue.ts b/ts/jobs/JobQueue.ts index 5391c4df2def..5a64eda31207 100644 --- a/ts/jobs/JobQueue.ts +++ b/ts/jobs/JobQueue.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-only import PQueue from 'p-queue'; -import { v4 as uuid } from 'uuid'; +import { v7 as uuid } from 'uuid'; import { noop } from 'lodash'; import { Job } from './Job'; diff --git a/ts/jobs/helpers/sendReaction.ts b/ts/jobs/helpers/sendReaction.ts index 86372b04a19e..508b9306a4a4 100644 --- a/ts/jobs/helpers/sendReaction.ts +++ b/ts/jobs/helpers/sendReaction.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: AGPL-3.0-only import { isNumber } from 'lodash'; -import { v4 as generateUuid } from 'uuid'; import * as Errors from '../../types/errors'; import { strictAssert } from '../../util/assert'; @@ -32,6 +31,7 @@ import type { AciString, ServiceIdString } from '../../types/ServiceId'; import { isAciString } from '../../util/isAciString'; import { handleMultipleSendErrors } from './handleMultipleSendErrors'; import { incrementMessageCounter } from '../../util/incrementMessageCounter'; +import { generateMessageId } from '../../util/generateMessageId'; import type { ConversationQueueJobBundle, @@ -159,11 +159,10 @@ export async function sendReaction( remove: !emoji, }; const ephemeralMessageForReactionSend = new window.Whisper.Message({ - id: generateUuid(), + ...generateMessageId(incrementMessageCounter()), type: 'outgoing', conversationId: conversation.get('id'), sent_at: pendingReaction.timestamp, - received_at: incrementMessageCounter(), received_at_ms: pendingReaction.timestamp, timestamp: pendingReaction.timestamp, sendStateByConversationId: zipObject( diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index 9f3568db48f8..2bde5b38d517 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -174,6 +174,7 @@ import { ReceiptType } from '../types/Receipt'; import { getQuoteAttachment } from '../util/makeQuote'; import { deriveProfileKeyVersion } from '../util/zkgroup'; import { incrementMessageCounter } from '../util/incrementMessageCounter'; +import { generateMessageId } from '../util/generateMessageId'; import { getMessageAuthorText } from '../util/getMessageAuthorText'; import { downscaleOutgoingAttachment } from '../util/attachments'; import { MessageRequestResponseEvent } from '../types/MessageRequestResponseEvent'; @@ -2328,11 +2329,10 @@ export class ConversationModel extends window.Backbone : lastMessageTimestamp; const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, type: 'message-request-response-event', sent_at: maybeLastMessageTimestamp, - received_at: incrementMessageCounter(), received_at_ms: maybeLastMessageTimestamp, readStatus: ReadStatus.Read, seenStatus: SeenStatus.NotApplicable, @@ -3085,12 +3085,11 @@ export class ConversationModel extends window.Backbone }); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(receivedAtCounter), conversationId: this.id, type: 'chat-session-refreshed', timestamp: receivedAt, sent_at: receivedAt, - received_at: receivedAtCounter, received_at_ms: receivedAt, readStatus: ReadStatus.Unread, seenStatus: SeenStatus.Unseen, @@ -3133,12 +3132,11 @@ export class ConversationModel extends window.Backbone } const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(receivedAtCounter), conversationId: this.id, type: 'delivery-issue', sourceServiceId: senderAci, sent_at: receivedAt, - received_at: receivedAtCounter, received_at_ms: receivedAt, timestamp: receivedAt, readStatus: ReadStatus.Unread, @@ -3183,12 +3181,11 @@ export class ConversationModel extends window.Backbone const timestamp = Date.now(); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, type: 'keychange', sent_at: timestamp, timestamp, - received_at: incrementMessageCounter(), received_at_ms: timestamp, key_changed: keyChangedId, readStatus: ReadStatus.Read, @@ -3245,12 +3242,11 @@ export class ConversationModel extends window.Backbone const timestamp = Date.now(); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, type: 'conversation-merge', sent_at: timestamp, timestamp, - received_at: incrementMessageCounter(), received_at_ms: timestamp, conversationMerge: { renderInfo, @@ -3294,12 +3290,11 @@ export class ConversationModel extends window.Backbone log.info(`${logId}: adding notification`); const timestamp = Date.now(); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, type: 'phone-number-discovery', sent_at: timestamp, timestamp, - received_at: incrementMessageCounter(), received_at_ms: timestamp, phoneNumberDiscovery: { e164, @@ -3343,12 +3338,11 @@ export class ConversationModel extends window.Backbone const timestamp = Date.now(); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, local: Boolean(options.local), readStatus: ReadStatus.Read, received_at_ms: timestamp, - received_at: incrementMessageCounter(), seenStatus: options.local ? SeenStatus.Seen : SeenStatus.Unseen, sent_at: lastMessage, timestamp, @@ -3388,11 +3382,10 @@ export class ConversationModel extends window.Backbone ): Promise { const now = Date.now(); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, type: 'profile-change', sent_at: now, - received_at: incrementMessageCounter(), received_at_ms: now, readStatus: ReadStatus.Read, seenStatus: SeenStatus.NotApplicable, @@ -3432,11 +3425,10 @@ export class ConversationModel extends window.Backbone ): Promise { const now = Date.now(); const message: MessageAttributesType = { - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: this.id, type, sent_at: now, - received_at: incrementMessageCounter(), received_at_ms: now, timestamp: now, @@ -4102,7 +4094,7 @@ export class ConversationModel extends window.Backbone // Here we move attachments to disk const attributes = await upgradeMessageSchema({ - id: generateGuid(), + ...generateMessageId(incrementMessageCounter()), timestamp: now, type: 'outgoing', body, @@ -4112,7 +4104,6 @@ export class ConversationModel extends window.Backbone preview, attachments: attachmentsToSend, sent_at: now, - received_at: incrementMessageCounter(), received_at_ms: now, expirationStartTimestamp, expireTimer, @@ -4734,9 +4725,9 @@ export class ConversationModel extends window.Backbone const shouldBeRead = (isInitialSync && isFromSyncOperation) || isFromMe || isNoteToSelf; - const id = generateGuid(); + const counter = receivedAt ?? incrementMessageCounter(); const attributes = { - id, + ...generateMessageId(counter), conversationId: this.id, expirationTimerUpdate: { expireTimer, @@ -4747,7 +4738,6 @@ export class ConversationModel extends window.Backbone flags: Proto.DataMessage.Flags.EXPIRATION_TIMER_UPDATE, readStatus: shouldBeRead ? ReadStatus.Read : ReadStatus.Unread, received_at_ms: receivedAtMS, - received_at: receivedAt ?? incrementMessageCounter(), seenStatus: shouldBeRead ? SeenStatus.Seen : SeenStatus.Unseen, sent_at: sentAt, timestamp: sentAt, @@ -4760,7 +4750,7 @@ export class ConversationModel extends window.Backbone }); window.MessageCache.__DEPRECATED$register( - id, + attributes.id, attributes, 'updateExpirationTimer' ); diff --git a/ts/models/messages.ts b/ts/models/messages.ts index 3029285eb94e..1afa16a95612 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -11,7 +11,6 @@ import { pick, union, } from 'lodash'; -import { v4 as generateUuid } from 'uuid'; import type { CustomError, @@ -26,6 +25,7 @@ import { isNotNil } from '../util/isNotNil'; import { isNormalNumber } from '../util/isNormalNumber'; import { strictAssert } from '../util/assert'; import { hydrateStoryContext } from '../util/hydrateStoryContext'; +import { generateMessageId } from '../util/generateMessageId'; import { drop } from '../util/drop'; import type { ConversationModel } from './conversations'; import type { @@ -1642,7 +1642,8 @@ export class MessageModel extends window.Backbone.Model { } } - const messageId = message.get('id') || generateUuid(); + const messageId = + message.get('id') || generateMessageId(this.get('received_at')).id; // Send delivery receipts, but only for non-story sealed sender messages // and not for messages from unaccepted conversations diff --git a/ts/reactions/enqueueReactionForSend.ts b/ts/reactions/enqueueReactionForSend.ts index 63c4174e23e7..6b936f8ed068 100644 --- a/ts/reactions/enqueueReactionForSend.ts +++ b/ts/reactions/enqueueReactionForSend.ts @@ -2,9 +2,10 @@ // SPDX-License-Identifier: AGPL-3.0-only import noop from 'lodash/noop'; -import { v4 as generateUuid } from 'uuid'; +import { v7 as generateUuid } from 'uuid'; import { DataWriter } from '../sql/Client'; +import type { MessageModel } from '../models/messages'; import type { ReactionAttributesType } from '../messageModifiers/Reactions'; import { ReactionSource } from './ReactionSource'; import { __DEPRECATED$getMessageById } from '../messages/getMessageById'; @@ -12,6 +13,7 @@ import { getSourceServiceId, isStory } from '../messages/helpers'; import { strictAssert } from '../util/assert'; import { isDirectConversation } from '../util/whatTypeOfConversation'; import { incrementMessageCounter } from '../util/incrementMessageCounter'; +import { generateMessageId } from '../util/generateMessageId'; import { repeat, zipObject } from '../util/iterables'; import { getMessageSentTimestamp } from '../util/getMessageSentTimestamp'; import { isAciString } from '../util/isAciString'; @@ -87,31 +89,31 @@ export async function enqueueReactionForSend({ : undefined; // Only used in story scenarios, where we use a whole message to represent the reaction - const storyReactionMessage = storyMessage - ? new window.Whisper.Message({ - id: generateUuid(), - type: 'outgoing', - conversationId: targetConversation.id, - sent_at: timestamp, - received_at: incrementMessageCounter(), - received_at_ms: timestamp, - timestamp, - expireTimer, - sendStateByConversationId: zipObject( - targetConversation.getMemberConversationIds(), - repeat({ - status: SendStatus.Pending, - updatedAt: Date.now(), - }) - ), - storyId: message.id, - storyReaction: { - emoji, - targetAuthorAci, - targetTimestamp, - }, - }) - : undefined; + let storyReactionMessage: MessageModel | undefined; + if (storyMessage) { + storyReactionMessage = new window.Whisper.Message({ + ...generateMessageId(incrementMessageCounter()), + type: 'outgoing', + conversationId: targetConversation.id, + sent_at: timestamp, + received_at_ms: timestamp, + timestamp, + expireTimer, + sendStateByConversationId: zipObject( + targetConversation.getMemberConversationIds(), + repeat({ + status: SendStatus.Pending, + updatedAt: Date.now(), + }) + ), + storyId: message.id, + storyReaction: { + emoji, + targetAuthorAci, + targetTimestamp, + }, + }); + } const reaction: ReactionAttributesType = { envelopeId: generateUuid(), diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index 860d120892cf..0f6f7111529c 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -3,7 +3,7 @@ import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client'; import { ReceiptCredentialPresentation } from '@signalapp/libsignal-client/zkgroup'; -import { v4 as generateUuid } from 'uuid'; +import { v7 as generateUuid } from 'uuid'; import pMap from 'p-map'; import { Writable } from 'stream'; import { isNumber } from 'lodash'; @@ -63,6 +63,7 @@ import { deriveGroupPublicParams, } from '../../util/zkgroup'; import { incrementMessageCounter } from '../../util/incrementMessageCounter'; +import { generateMessageId } from '../../util/generateMessageId'; import { isAciString } from '../../util/isAciString'; import { PhoneNumberDiscoverability } from '../../util/phoneNumberDiscoverability'; import { PhoneNumberSharingMode } from '../../util/phoneNumberSharingMode'; @@ -487,6 +488,26 @@ export class BackupImportStream extends Writable { const batch = Array.from(this.saveMessageBatch); this.saveMessageBatch.clear(); + // There are a few indexes that start with message id, and many more that + // start with conversationId. Sort messages by both to make sure that we + // are not doing random insertions into the database file. + // This improves bulk insert performance >2x. + batch.sort((a, b) => { + if (a.conversationId > b.conversationId) { + return -1; + } + if (a.conversationId < b.conversationId) { + return 1; + } + if (a.id < b.id) { + return -1; + } + if (a.id > b.id) { + return 1; + } + return 0; + }); + await DataWriter.saveMessages(batch, { forceSave: true, ourAci, @@ -1235,9 +1256,8 @@ export class BackupImportStream extends Writable { } let attributes: MessageAttributesType = { - id: generateUuid(), + ...generateMessageId(incrementMessageCounter()), conversationId: chatConvo.id, - received_at: incrementMessageCounter(), sent_at: timestamp, source: authorConvo?.e164, sourceServiceId: authorConvo?.serviceId, diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index d189c510c85a..891beffae5aa 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -18,7 +18,7 @@ import { cleanDataForIpc } from './cleanDataForIpc'; import type { AciString, ServiceIdString } from '../types/ServiceId'; import createTaskWithTimeout from '../textsecure/TaskWithTimeout'; import * as log from '../logging/log'; -import { isValidUuid } from '../util/isValidUuid'; +import { isValidUuidV7 } from '../util/isValidUuid'; import * as Errors from '../types/errors'; import type { StoredJob } from '../jobs/types'; @@ -603,7 +603,7 @@ async function saveMessage( jobToInsert: options.jobToInsert && formatJobForInsert(options.jobToInsert), }); - softAssert(isValidUuid(id), 'saveMessage: messageId is not a UUID'); + softAssert(isValidUuidV7(id), 'saveMessage: messageId is not a UUID'); void updateExpiringMessagesService(); void tapToViewMessagesDeletionService.update(); diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index a712dcb5568a..16d29d4996eb 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -9,7 +9,6 @@ import rimraf from 'rimraf'; import { randomBytes } from 'crypto'; import type { Database, Statement } from '@signalapp/better-sqlite3'; import SQL from '@signalapp/better-sqlite3'; -import { v4 as generateUuid } from 'uuid'; import { z } from 'zod'; import type { ReadonlyDeep } from 'type-fest'; @@ -49,6 +48,7 @@ import { isNormalNumber } from '../util/isNormalNumber'; import { isNotNil } from '../util/isNotNil'; import { parseIntOrThrow } from '../util/parseIntOrThrow'; import * as durations from '../util/durations'; +import { generateMessageId } from '../util/generateMessageId'; import { formatCountForLogging } from '../logging/formatCountForLogging'; import type { ConversationColorType, CustomColorType } from '../types/Colors'; import type { BadgeType, BadgeImageType } from '../badges/types'; @@ -2328,7 +2328,7 @@ export function saveMessage( const toCreate = { ...data, - id: id || generateUuid(), + id: id || generateMessageId(data.received_at).id, }; prepare( @@ -6813,23 +6813,30 @@ function pageMessages( writable.exec( ` CREATE TEMP TABLE tmp_${runId}_updated_messages - (rowid INTEGER PRIMARY KEY ASC); + (rowid INTEGER PRIMARY KEY, received_at INTEGER, sent_at INTEGER); - INSERT INTO tmp_${runId}_updated_messages (rowid) - SELECT rowid FROM messages; + CREATE INDEX tmp_${runId}_updated_messages_received_at + ON tmp_${runId}_updated_messages (received_at ASC, sent_at ASC); + + INSERT INTO tmp_${runId}_updated_messages + (rowid, received_at, sent_at) + SELECT rowid, received_at, sent_at FROM messages + ORDER BY received_at ASC, sent_at ASC; CREATE TEMP TRIGGER tmp_${runId}_message_updates UPDATE OF json ON messages BEGIN - INSERT OR IGNORE INTO tmp_${runId}_updated_messages (rowid) - VALUES (NEW.rowid); + INSERT OR IGNORE INTO tmp_${runId}_updated_messages + (rowid, received_at, sent_at) + VALUES (NEW.rowid, NEW.received_at, NEW.sent_at); END; CREATE TEMP TRIGGER tmp_${runId}_message_inserts AFTER INSERT ON messages BEGIN - INSERT OR IGNORE INTO tmp_${runId}_updated_messages (rowid) - VALUES (NEW.rowid); + INSERT OR IGNORE INTO tmp_${runId}_updated_messages + (rowid, received_at, sent_at) + VALUES (NEW.rowid, NEW.received_at, NEW.sent_at); END; ` ); @@ -6840,10 +6847,11 @@ function pageMessages( const rowids: Array = writable .prepare( ` - DELETE FROM tmp_${runId}_updated_messages - RETURNING rowid - LIMIT $chunkSize; - ` + DELETE FROM tmp_${runId}_updated_messages + RETURNING rowid + ORDER BY received_at ASC, sent_at ASC + LIMIT $chunkSize; + ` ) .pluck() .all({ chunkSize }); diff --git a/ts/test-electron/MessageReceipts_test.ts b/ts/test-electron/MessageReceipts_test.ts index 1871161e7768..65f23e3218b6 100644 --- a/ts/test-electron/MessageReceipts_test.ts +++ b/ts/test-electron/MessageReceipts_test.ts @@ -1,7 +1,7 @@ // Copyright 2023 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import { assert } from 'chai'; import { type AciString, generateAci } from '../types/ServiceId'; diff --git a/ts/test-node/util/callDisposition_test.ts b/ts/test-node/util/callDisposition_test.ts index 0b59321fb90c..69769f958603 100644 --- a/ts/test-node/util/callDisposition_test.ts +++ b/ts/test-node/util/callDisposition_test.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import type { PeekInfo } from '@signalapp/ringrtc'; -import uuid from 'uuid'; +import { v4 as uuid } from 'uuid'; import { getPeerIdFromConversation, getCallIdFromEra, diff --git a/ts/util/generateMessageId.ts b/ts/util/generateMessageId.ts new file mode 100644 index 000000000000..f3ed6a56e470 --- /dev/null +++ b/ts/util/generateMessageId.ts @@ -0,0 +1,47 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { stringify } from 'uuid'; + +import { getRandomBytes } from '../Crypto'; + +export type GeneratedMessageIdType = Readonly<{ + id: string; + received_at: number; +}>; + +// See https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis-00#section-5.7 +export function generateMessageId(counter: number): GeneratedMessageIdType { + const uuid = getRandomBytes(16); + + /* eslint-disable no-bitwise */ + + // We compose uuid out of 48 bits (6 bytes of) timestamp-like counter: + // `incrementMessageCounter`. Note big-endian encoding (which ensures proper + // lexicographical order), and floating point divisions (because `&` operator + // coerces to 32bit integers) + + uuid[0] = (counter / 0x10000000000) & 0xff; + uuid[1] = (counter / 0x00100000000) & 0xff; + uuid[2] = (counter / 0x00001000000) & 0xff; + uuid[3] = (counter / 0x00000010000) & 0xff; + uuid[4] = (counter / 0x00000000100) & 0xff; + uuid[5] = (counter / 0x00000000001) & 0xff; + + // Mask out 4 bits of version number + uuid[6] &= 0x0f; + // And set the version to 7 + uuid[6] |= 0x70; + + // Mask out 2 bits of variant + uuid[8] &= 0x3f; + // And set it to "2" + uuid[8] |= 0x80; + + /* eslint-enable no-bitwise */ + + return { + id: stringify(uuid), + received_at: counter, + }; +} diff --git a/ts/util/incrementMessageCounter.ts b/ts/util/incrementMessageCounter.ts index 0ddc17330d2a..d33a3aac8b33 100644 --- a/ts/util/incrementMessageCounter.ts +++ b/ts/util/incrementMessageCounter.ts @@ -4,6 +4,7 @@ import { debounce, isNumber } from 'lodash'; import { strictAssert } from './assert'; +import { safeParseInteger } from './numbers'; import { DataReader } from '../sql/Client'; import * as log from '../logging/log'; @@ -15,7 +16,9 @@ export async function initializeMessageCounter(): Promise { 'incrementMessageCounter: already initialized' ); - const storedCounter = Number(localStorage.getItem('lastReceivedAtCounter')); + const storedCounter = safeParseInteger( + localStorage.getItem('lastReceivedAtCounter') ?? '' + ); const dbCounter = await DataReader.getMaxMessageCounter(); if (isNumber(dbCounter) && isNumber(storedCounter)) { diff --git a/ts/util/isValidUuid.ts b/ts/util/isValidUuid.ts index bcb4a9d53c26..526a0ade3050 100644 --- a/ts/util/isValidUuid.ts +++ b/ts/util/isValidUuid.ts @@ -16,3 +16,19 @@ export const isValidUuid = (value: unknown): value is string => { return UUID_REGEXP.test(value); }; + +const UUID_V7_REGEXP = + /^[0-9A-F]{8}-[0-9A-F]{4}-7[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i; + +export const isValidUuidV7 = (value: unknown): value is string => { + if (typeof value !== 'string') { + return false; + } + + // Zero UUID is a valid uuid. + if (value === '00000000-0000-0000-0000-000000000000') { + return true; + } + + return UUID_V7_REGEXP.test(value); +}; diff --git a/ts/windows/attachments.ts b/ts/windows/attachments.ts index acd5fcfd4d31..680d77ad8884 100644 --- a/ts/windows/attachments.ts +++ b/ts/windows/attachments.ts @@ -5,7 +5,7 @@ import { ipcRenderer } from 'electron'; import { isString, isTypedArray } from 'lodash'; import { join, normalize, basename } from 'path'; import fse from 'fs-extra'; -import getGuid from 'uuid/v4'; +import { v4 as getGuid } from 'uuid'; import { isPathInside } from '../util/isPathInside'; import { writeWindowsZoneIdentifier } from '../util/windowsZoneIdentifier';