Use UUIDs in group database schema
This commit is contained in:
Fedor Indutny
GitHub
parent
74fde10ff5
commit
63fcdbe787
79 changed files with 4530 additions and 3664 deletions
|
|
@ -33,6 +33,7 @@ import { assert, strictAssert } from '../util/assert';
|
|||
import { cleanDataForIpc } from './cleanDataForIpc';
|
||||
import type { ReactionType } from '../types/Reactions';
|
||||
import type { ConversationColorType, CustomColorType } from '../types/Colors';
|
||||
import type { UUIDStringType } from '../types/UUID';
|
||||
import type { ProcessGroupCallRingRequestResult } from '../types/Calling';
|
||||
import type { RemoveAllConfiguration } from '../types/RemoveAllConfiguration';
|
||||
import createTaskWithTimeout from '../textsecure/TaskWithTimeout';
|
||||
|
|
@ -204,7 +205,7 @@ const dataInterface: ClientInterface = {
|
|||
getAllConversations,
|
||||
getAllConversationIds,
|
||||
getAllPrivateConversations,
|
||||
getAllGroupsInvolvingId,
|
||||
getAllGroupsInvolvingUuid,
|
||||
|
||||
searchConversations,
|
||||
searchMessages,
|
||||
|
|
@ -1044,15 +1045,15 @@ async function getAllPrivateConversations({
|
|||
return collection;
|
||||
}
|
||||
|
||||
async function getAllGroupsInvolvingId(
|
||||
id: string,
|
||||
async function getAllGroupsInvolvingUuid(
|
||||
uuid: UUIDStringType,
|
||||
{
|
||||
ConversationCollection,
|
||||
}: {
|
||||
ConversationCollection: typeof ConversationModelCollectionType;
|
||||
}
|
||||
) {
|
||||
const conversations = await channels.getAllGroupsInvolvingId(id);
|
||||
const conversations = await channels.getAllGroupsInvolvingUuid(uuid);
|
||||
|
||||
const collection = new ConversationCollection();
|
||||
collection.add(conversations);
|
||||
|
|
@ -1325,11 +1326,11 @@ async function getNewerMessagesByConversation(
|
|||
}
|
||||
async function getLastConversationMessages({
|
||||
conversationId,
|
||||
ourConversationId,
|
||||
ourUuid,
|
||||
Message,
|
||||
}: {
|
||||
conversationId: string;
|
||||
ourConversationId: string;
|
||||
ourUuid: UUIDStringType;
|
||||
Message: typeof MessageModel;
|
||||
}): Promise<LastConversationMessagesType> {
|
||||
const {
|
||||
|
|
@ -1338,7 +1339,7 @@ async function getLastConversationMessages({
|
|||
hasUserInitiatedMessages,
|
||||
} = await channels.getLastConversationMessages({
|
||||
conversationId,
|
||||
ourConversationId,
|
||||
ourUuid,
|
||||
});
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ export type EmojiType = {
|
|||
|
||||
export type IdentityKeyType = {
|
||||
firstUse: boolean;
|
||||
id: UUIDStringType | `conversation:${UUIDStringType}`;
|
||||
id: UUIDStringType | `conversation:${string}`;
|
||||
nonblockingApproval: boolean;
|
||||
publicKey: Uint8Array;
|
||||
timestamp: number;
|
||||
|
|
@ -501,7 +501,9 @@ export type DataInterface = {
|
|||
|
||||
export type ServerInterface = DataInterface & {
|
||||
getAllConversations: () => Promise<Array<ConversationType>>;
|
||||
getAllGroupsInvolvingId: (id: string) => Promise<Array<ConversationType>>;
|
||||
getAllGroupsInvolvingUuid: (
|
||||
id: UUIDStringType
|
||||
) => Promise<Array<ConversationType>>;
|
||||
getAllPrivateConversations: () => Promise<Array<ConversationType>>;
|
||||
getConversationById: (id: string) => Promise<ConversationType | undefined>;
|
||||
getExpiredMessages: () => Promise<Array<MessageType>>;
|
||||
|
|
@ -528,7 +530,7 @@ export type ServerInterface = DataInterface & {
|
|||
) => Promise<Array<MessageTypeUnhydrated>>;
|
||||
getLastConversationMessages: (options: {
|
||||
conversationId: string;
|
||||
ourConversationId: string;
|
||||
ourUuid: UUIDStringType;
|
||||
}) => Promise<LastConversationMessagesServerType>;
|
||||
getTapToViewMessagesNeedingErase: () => Promise<Array<MessageType>>;
|
||||
removeConversation: (id: Array<string> | string) => Promise<void>;
|
||||
|
|
@ -576,8 +578,8 @@ export type ClientInterface = DataInterface & {
|
|||
getAllConversations: (options: {
|
||||
ConversationCollection: typeof ConversationModelCollectionType;
|
||||
}) => Promise<ConversationModelCollectionType>;
|
||||
getAllGroupsInvolvingId: (
|
||||
id: string,
|
||||
getAllGroupsInvolvingUuid: (
|
||||
id: UUIDStringType,
|
||||
options: {
|
||||
ConversationCollection: typeof ConversationModelCollectionType;
|
||||
}
|
||||
|
|
@ -630,7 +632,7 @@ export type ClientInterface = DataInterface & {
|
|||
) => Promise<MessageModelCollectionType>;
|
||||
getLastConversationMessages: (options: {
|
||||
conversationId: string;
|
||||
ourConversationId: string;
|
||||
ourUuid: UUIDStringType;
|
||||
Message: typeof MessageModel;
|
||||
}) => Promise<LastConversationMessagesType>;
|
||||
getTapToViewMessagesNeedingErase: (options: {
|
||||
|
|
|
|||
2778
ts/sql/Server.ts
2778
ts/sql/Server.ts
File diff suppressed because it is too large
Load diff
448
ts/sql/migrations/41-uuid-keys.ts
Normal file
448
ts/sql/migrations/41-uuid-keys.ts
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
// Copyright 2021 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import type { Database } from 'better-sqlite3';
|
||||
|
||||
import type { LoggerType } from '../../types/Logging';
|
||||
import { isValidUuid } from '../../types/UUID';
|
||||
import { assertSync } from '../../util/assert';
|
||||
import Helpers from '../../textsecure/Helpers';
|
||||
import { createOrUpdate, getById, removeById } from '../util';
|
||||
import type { EmptyQuery, Query } from '../util';
|
||||
import type { ItemKeyType } from '../Interface';
|
||||
|
||||
function getOurUuid(db: Database): string | undefined {
|
||||
const UUID_ID: ItemKeyType = 'uuid_id';
|
||||
|
||||
const row: { json: string } | undefined = db
|
||||
.prepare<Query>('SELECT json FROM items WHERE id = $id;')
|
||||
.get({ id: UUID_ID });
|
||||
|
||||
if (!row) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const { value } = JSON.parse(row.json);
|
||||
|
||||
const [ourUuid] = Helpers.unencodeNumber(String(value).toLowerCase());
|
||||
return ourUuid;
|
||||
}
|
||||
|
||||
export default function updateToSchemaVersion41(
|
||||
currentVersion: number,
|
||||
db: Database,
|
||||
logger: LoggerType
|
||||
): void {
|
||||
if (currentVersion >= 41) {
|
||||
return;
|
||||
}
|
||||
|
||||
const getConversationUuid = db
|
||||
.prepare<Query>(
|
||||
`
|
||||
SELECT uuid
|
||||
FROM
|
||||
conversations
|
||||
WHERE
|
||||
id = $conversationId
|
||||
`
|
||||
)
|
||||
.pluck();
|
||||
|
||||
const getConversationStats = db.prepare<Query>(
|
||||
`
|
||||
SELECT uuid, e164, active_at
|
||||
FROM
|
||||
conversations
|
||||
WHERE
|
||||
id = $conversationId
|
||||
`
|
||||
);
|
||||
|
||||
const compareConvoRecency = (a: string, b: string): number => {
|
||||
const aStats = getConversationStats.get({ conversationId: a });
|
||||
const bStats = getConversationStats.get({ conversationId: b });
|
||||
|
||||
const isAComplete = Boolean(aStats?.uuid && aStats?.e164);
|
||||
const isBComplete = Boolean(bStats?.uuid && bStats?.e164);
|
||||
|
||||
if (!isAComplete && !isBComplete) {
|
||||
return 0;
|
||||
}
|
||||
if (!isAComplete) {
|
||||
return -1;
|
||||
}
|
||||
if (!isBComplete) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return aStats.active_at - bStats.active_at;
|
||||
};
|
||||
|
||||
const clearSessionsAndKeys = () => {
|
||||
// ts/background.ts will ask user to relink so all that matters here is
|
||||
// to maintain an invariant:
|
||||
//
|
||||
// After this migration all sessions and keys are prefixed by
|
||||
// "uuid:".
|
||||
db.exec(
|
||||
`
|
||||
DELETE FROM senderKeys;
|
||||
DELETE FROM sessions;
|
||||
DELETE FROM signedPreKeys;
|
||||
DELETE FROM preKeys;
|
||||
`
|
||||
);
|
||||
|
||||
assertSync(removeById<string>(db, 'items', 'identityKey'));
|
||||
assertSync(removeById<string>(db, 'items', 'registrationId'));
|
||||
};
|
||||
|
||||
const moveIdentityKeyToMap = (ourUuid: string) => {
|
||||
type IdentityKeyType = {
|
||||
privKey: string;
|
||||
publicKey: string;
|
||||
};
|
||||
|
||||
const identityKey = assertSync(
|
||||
getById<string, { value: IdentityKeyType }>(db, 'items', 'identityKey')
|
||||
);
|
||||
|
||||
type RegistrationId = number;
|
||||
|
||||
const registrationId = assertSync(
|
||||
getById<string, { value: RegistrationId }>(db, 'items', 'registrationId')
|
||||
);
|
||||
|
||||
if (identityKey) {
|
||||
assertSync(
|
||||
createOrUpdate<ItemKeyType>(db, 'items', {
|
||||
id: 'identityKeyMap',
|
||||
value: {
|
||||
[ourUuid]: identityKey.value,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
if (registrationId) {
|
||||
assertSync(
|
||||
createOrUpdate<ItemKeyType>(db, 'items', {
|
||||
id: 'registrationIdMap',
|
||||
value: {
|
||||
[ourUuid]: registrationId.value,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
db.exec(
|
||||
`
|
||||
DELETE FROM items WHERE id = "identityKey" OR id = "registrationId";
|
||||
`
|
||||
);
|
||||
};
|
||||
|
||||
const prefixKeys = (ourUuid: string) => {
|
||||
for (const table of ['signedPreKeys', 'preKeys']) {
|
||||
// Update id to include suffix, add `ourUuid` and `keyId` fields.
|
||||
db.prepare<Query>(
|
||||
`
|
||||
UPDATE ${table}
|
||||
SET
|
||||
id = $ourUuid || ':' || id,
|
||||
json = json_set(
|
||||
json,
|
||||
'$.id',
|
||||
$ourUuid || ':' || json_extract(json, '$.id'),
|
||||
'$.keyId',
|
||||
json_extract(json, '$.id'),
|
||||
'$.ourUuid',
|
||||
$ourUuid
|
||||
)
|
||||
`
|
||||
).run({ ourUuid });
|
||||
}
|
||||
};
|
||||
|
||||
const updateSenderKeys = (ourUuid: string) => {
|
||||
const senderKeys: ReadonlyArray<{
|
||||
id: string;
|
||||
senderId: string;
|
||||
lastUpdatedDate: number;
|
||||
}> = db
|
||||
.prepare<EmptyQuery>(
|
||||
'SELECT id, senderId, lastUpdatedDate FROM senderKeys'
|
||||
)
|
||||
.all();
|
||||
|
||||
logger.info(`Updating ${senderKeys.length} sender keys`);
|
||||
|
||||
const updateSenderKey = db.prepare<Query>(
|
||||
`
|
||||
UPDATE senderKeys
|
||||
SET
|
||||
id = $newId,
|
||||
senderId = $newSenderId
|
||||
WHERE
|
||||
id = $id
|
||||
`
|
||||
);
|
||||
|
||||
const deleteSenderKey = db.prepare<Query>(
|
||||
'DELETE FROM senderKeys WHERE id = $id'
|
||||
);
|
||||
|
||||
const pastKeys = new Map<
|
||||
string,
|
||||
{
|
||||
conversationId: string;
|
||||
lastUpdatedDate: number;
|
||||
}
|
||||
>();
|
||||
|
||||
let updated = 0;
|
||||
let deleted = 0;
|
||||
let skipped = 0;
|
||||
for (const { id, senderId, lastUpdatedDate } of senderKeys) {
|
||||
const [conversationId] = Helpers.unencodeNumber(senderId);
|
||||
const uuid = getConversationUuid.get({ conversationId });
|
||||
|
||||
if (!uuid) {
|
||||
deleted += 1;
|
||||
deleteSenderKey.run({ id });
|
||||
continue;
|
||||
}
|
||||
|
||||
const newId = `${ourUuid}:${id.replace(conversationId, uuid)}`;
|
||||
|
||||
const existing = pastKeys.get(newId);
|
||||
|
||||
// We are going to delete on of the keys anyway
|
||||
if (existing) {
|
||||
skipped += 1;
|
||||
} else {
|
||||
updated += 1;
|
||||
}
|
||||
|
||||
const isOlder =
|
||||
existing &&
|
||||
(lastUpdatedDate < existing.lastUpdatedDate ||
|
||||
compareConvoRecency(conversationId, existing.conversationId) < 0);
|
||||
if (isOlder) {
|
||||
deleteSenderKey.run({ id });
|
||||
continue;
|
||||
} else if (existing) {
|
||||
deleteSenderKey.run({ id: newId });
|
||||
}
|
||||
|
||||
pastKeys.set(newId, { conversationId, lastUpdatedDate });
|
||||
|
||||
updateSenderKey.run({
|
||||
id,
|
||||
newId,
|
||||
newSenderId: `${senderId.replace(conversationId, uuid)}`,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Updated ${senderKeys.length} sender keys: ` +
|
||||
`updated: ${updated}, deleted: ${deleted}, skipped: ${skipped}`
|
||||
);
|
||||
};
|
||||
|
||||
const updateSessions = (ourUuid: string) => {
|
||||
// Use uuid instead of conversation id in existing sesions and prefix id
|
||||
// with ourUuid.
|
||||
//
|
||||
// Set ourUuid column and field in json
|
||||
const allSessions = db
|
||||
.prepare<EmptyQuery>('SELECT id, conversationId FROM SESSIONS')
|
||||
.all();
|
||||
|
||||
logger.info(`Updating ${allSessions.length} sessions`);
|
||||
|
||||
const updateSession = db.prepare<Query>(
|
||||
`
|
||||
UPDATE sessions
|
||||
SET
|
||||
id = $newId,
|
||||
ourUuid = $ourUuid,
|
||||
uuid = $uuid,
|
||||
json = json_set(
|
||||
sessions.json,
|
||||
'$.id',
|
||||
$newId,
|
||||
'$.uuid',
|
||||
$uuid,
|
||||
'$.ourUuid',
|
||||
$ourUuid
|
||||
)
|
||||
WHERE
|
||||
id = $id
|
||||
`
|
||||
);
|
||||
|
||||
const deleteSession = db.prepare<Query>(
|
||||
'DELETE FROM sessions WHERE id = $id'
|
||||
);
|
||||
|
||||
const pastSessions = new Map<
|
||||
string,
|
||||
{
|
||||
conversationId: string;
|
||||
}
|
||||
>();
|
||||
|
||||
let updated = 0;
|
||||
let deleted = 0;
|
||||
let skipped = 0;
|
||||
for (const { id, conversationId } of allSessions) {
|
||||
const uuid = getConversationUuid.get({ conversationId });
|
||||
if (!uuid) {
|
||||
deleted += 1;
|
||||
deleteSession.run({ id });
|
||||
continue;
|
||||
}
|
||||
|
||||
const newId = `${ourUuid}:${id.replace(conversationId, uuid)}`;
|
||||
|
||||
const existing = pastSessions.get(newId);
|
||||
|
||||
// We are going to delete on of the keys anyway
|
||||
if (existing) {
|
||||
skipped += 1;
|
||||
} else {
|
||||
updated += 1;
|
||||
}
|
||||
|
||||
const isOlder =
|
||||
existing &&
|
||||
compareConvoRecency(conversationId, existing.conversationId) < 0;
|
||||
if (isOlder) {
|
||||
deleteSession.run({ id });
|
||||
continue;
|
||||
} else if (existing) {
|
||||
deleteSession.run({ id: newId });
|
||||
}
|
||||
|
||||
pastSessions.set(newId, { conversationId });
|
||||
|
||||
updateSession.run({
|
||||
id,
|
||||
newId,
|
||||
uuid,
|
||||
ourUuid,
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Updated ${allSessions.length} sessions: ` +
|
||||
`updated: ${updated}, deleted: ${deleted}, skipped: ${skipped}`
|
||||
);
|
||||
};
|
||||
|
||||
const updateIdentityKeys = () => {
|
||||
const identityKeys: ReadonlyArray<{
|
||||
id: string;
|
||||
}> = db.prepare<EmptyQuery>('SELECT id FROM identityKeys').all();
|
||||
|
||||
logger.info(`Updating ${identityKeys.length} identity keys`);
|
||||
|
||||
const updateIdentityKey = db.prepare<Query>(
|
||||
`
|
||||
UPDATE identityKeys
|
||||
SET
|
||||
id = $newId,
|
||||
json = json_set(
|
||||
identityKeys.json,
|
||||
'$.id',
|
||||
$newId
|
||||
)
|
||||
WHERE
|
||||
id = $id
|
||||
`
|
||||
);
|
||||
|
||||
let migrated = 0;
|
||||
for (const { id } of identityKeys) {
|
||||
const uuid = getConversationUuid.get({ conversationId: id });
|
||||
|
||||
let newId: string;
|
||||
if (uuid) {
|
||||
migrated += 1;
|
||||
newId = uuid;
|
||||
} else {
|
||||
newId = `conversation:${id}`;
|
||||
}
|
||||
|
||||
updateIdentityKey.run({ id, newId });
|
||||
}
|
||||
|
||||
logger.info(`Migrated ${migrated} identity keys`);
|
||||
};
|
||||
|
||||
db.transaction(() => {
|
||||
db.exec(
|
||||
`
|
||||
-- Change type of 'id' column from INTEGER to STRING
|
||||
|
||||
ALTER TABLE preKeys
|
||||
RENAME TO old_preKeys;
|
||||
|
||||
ALTER TABLE signedPreKeys
|
||||
RENAME TO old_signedPreKeys;
|
||||
|
||||
CREATE TABLE preKeys(
|
||||
id STRING PRIMARY KEY ASC,
|
||||
json TEXT
|
||||
);
|
||||
CREATE TABLE signedPreKeys(
|
||||
id STRING PRIMARY KEY ASC,
|
||||
json TEXT
|
||||
);
|
||||
|
||||
-- sqlite handles the type conversion
|
||||
INSERT INTO preKeys SELECT * FROM old_preKeys;
|
||||
INSERT INTO signedPreKeys SELECT * FROM old_signedPreKeys;
|
||||
|
||||
DROP TABLE old_preKeys;
|
||||
DROP TABLE old_signedPreKeys;
|
||||
|
||||
-- Alter sessions
|
||||
|
||||
ALTER TABLE sessions
|
||||
ADD COLUMN ourUuid STRING;
|
||||
|
||||
ALTER TABLE sessions
|
||||
ADD COLUMN uuid STRING;
|
||||
`
|
||||
);
|
||||
|
||||
const ourUuid = getOurUuid(db);
|
||||
|
||||
if (!isValidUuid(ourUuid)) {
|
||||
logger.error(
|
||||
'updateToSchemaVersion41: no uuid is available clearing sessions'
|
||||
);
|
||||
|
||||
clearSessionsAndKeys();
|
||||
|
||||
db.pragma('user_version = 41');
|
||||
return;
|
||||
}
|
||||
|
||||
prefixKeys(ourUuid);
|
||||
|
||||
updateSenderKeys(ourUuid);
|
||||
|
||||
updateSessions(ourUuid);
|
||||
|
||||
moveIdentityKeyToMap(ourUuid);
|
||||
|
||||
updateIdentityKeys();
|
||||
|
||||
db.pragma('user_version = 41');
|
||||
})();
|
||||
logger.info('updateToSchemaVersion41: success!');
|
||||
}
|
||||
77
ts/sql/migrations/42-stale-reactions.ts
Normal file
77
ts/sql/migrations/42-stale-reactions.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
// Copyright 2021 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import type { Database } from 'better-sqlite3';
|
||||
|
||||
import { batchMultiVarQuery } from '../util';
|
||||
import type { ArrayQuery } from '../util';
|
||||
import type { LoggerType } from '../../types/Logging';
|
||||
|
||||
export default function updateToSchemaVersion42(
|
||||
currentVersion: number,
|
||||
db: Database,
|
||||
logger: LoggerType
|
||||
): void {
|
||||
if (currentVersion >= 42) {
|
||||
return;
|
||||
}
|
||||
|
||||
db.transaction(() => {
|
||||
// First, recreate messages table delete trigger with reaction support
|
||||
|
||||
db.exec(`
|
||||
DROP TRIGGER messages_on_delete;
|
||||
|
||||
CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
|
||||
DELETE FROM messages_fts WHERE rowid = old.rowid;
|
||||
DELETE FROM sendLogPayloads WHERE id IN (
|
||||
SELECT payloadId FROM sendLogMessageIds
|
||||
WHERE messageId = old.id
|
||||
);
|
||||
DELETE FROM reactions WHERE rowid IN (
|
||||
SELECT rowid FROM reactions
|
||||
WHERE messageId = old.id
|
||||
);
|
||||
END;
|
||||
`);
|
||||
|
||||
// Then, delete previously-orphaned reactions
|
||||
|
||||
// Note: we use `pluck` here to fetch only the first column of
|
||||
// returned row.
|
||||
const messageIdList: Array<string> = db
|
||||
.prepare('SELECT id FROM messages ORDER BY id ASC;')
|
||||
.pluck()
|
||||
.all();
|
||||
const allReactions: Array<{
|
||||
rowid: number;
|
||||
messageId: string;
|
||||
}> = db.prepare('SELECT rowid, messageId FROM reactions;').all();
|
||||
|
||||
const messageIds = new Set(messageIdList);
|
||||
const reactionsToDelete: Array<number> = [];
|
||||
|
||||
allReactions.forEach(reaction => {
|
||||
if (!messageIds.has(reaction.messageId)) {
|
||||
reactionsToDelete.push(reaction.rowid);
|
||||
}
|
||||
});
|
||||
|
||||
function deleteReactions(rowids: Array<number>) {
|
||||
db.prepare<ArrayQuery>(
|
||||
`
|
||||
DELETE FROM reactions
|
||||
WHERE rowid IN ( ${rowids.map(() => '?').join(', ')} );
|
||||
`
|
||||
).run(rowids);
|
||||
}
|
||||
|
||||
if (reactionsToDelete.length > 0) {
|
||||
logger.info(`Deleting ${reactionsToDelete.length} orphaned reactions`);
|
||||
batchMultiVarQuery(db, reactionsToDelete, deleteReactions);
|
||||
}
|
||||
|
||||
db.pragma('user_version = 42');
|
||||
})();
|
||||
logger.info('updateToSchemaVersion42: success!');
|
||||
}
|
||||
417
ts/sql/migrations/43-gv2-uuid.ts
Normal file
417
ts/sql/migrations/43-gv2-uuid.ts
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
// Copyright 2021 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import type { Database } from 'better-sqlite3';
|
||||
import { omit } from 'lodash';
|
||||
|
||||
import type { LoggerType } from '../../types/Logging';
|
||||
import { UUID } from '../../types/UUID';
|
||||
import type { UUIDStringType } from '../../types/UUID';
|
||||
import { isNotNil } from '../../util/isNotNil';
|
||||
import { assert } from '../../util/assert';
|
||||
import {
|
||||
TableIterator,
|
||||
getCountFromTable,
|
||||
jsonToObject,
|
||||
objectToJSON,
|
||||
} from '../util';
|
||||
import type { EmptyQuery, Query } from '../util';
|
||||
import type { MessageType, ConversationType } from '../Interface';
|
||||
|
||||
export default function updateToSchemaVersion43(
|
||||
currentVersion: number,
|
||||
db: Database,
|
||||
logger: LoggerType
|
||||
): void {
|
||||
if (currentVersion >= 43) {
|
||||
return;
|
||||
}
|
||||
|
||||
type LegacyPendingMemberType = {
|
||||
addedByUserId?: string;
|
||||
conversationId: string;
|
||||
};
|
||||
|
||||
type LegacyAdminApprovalType = {
|
||||
conversationId: string;
|
||||
};
|
||||
|
||||
type LegacyConversationType = {
|
||||
id: string;
|
||||
membersV2?: Array<{
|
||||
conversationId: string;
|
||||
}>;
|
||||
pendingMembersV2?: Array<LegacyPendingMemberType>;
|
||||
pendingAdminApprovalV2?: Array<LegacyAdminApprovalType>;
|
||||
};
|
||||
|
||||
const getConversationUuid = db
|
||||
.prepare<Query>(
|
||||
`
|
||||
SELECT uuid
|
||||
FROM
|
||||
conversations
|
||||
WHERE
|
||||
id = $conversationId
|
||||
`
|
||||
)
|
||||
.pluck();
|
||||
|
||||
const updateConversationStmt = db.prepare(
|
||||
`
|
||||
UPDATE conversations SET
|
||||
json = $json,
|
||||
members = $members
|
||||
WHERE id = $id;
|
||||
`
|
||||
);
|
||||
|
||||
const updateMessageStmt = db.prepare(
|
||||
`
|
||||
UPDATE messages SET
|
||||
json = $json,
|
||||
sourceUuid = $sourceUuid
|
||||
WHERE id = $id;
|
||||
`
|
||||
);
|
||||
|
||||
const upgradeConversation = (convo: ConversationType) => {
|
||||
const legacy = (convo as unknown) as LegacyConversationType;
|
||||
let result = convo;
|
||||
|
||||
const memberKeys: Array<keyof LegacyConversationType> = [
|
||||
'membersV2',
|
||||
'pendingMembersV2',
|
||||
'pendingAdminApprovalV2',
|
||||
];
|
||||
for (const key of memberKeys) {
|
||||
const oldValue = legacy[key];
|
||||
if (!Array.isArray(oldValue)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let addedByCount = 0;
|
||||
|
||||
const newValue = oldValue
|
||||
.map(member => {
|
||||
const uuid: UUIDStringType = getConversationUuid.get({
|
||||
conversationId: member.conversationId,
|
||||
});
|
||||
if (!uuid) {
|
||||
logger.warn(
|
||||
`updateToSchemaVersion43: ${legacy.id}.${key} UUID not found ` +
|
||||
`for ${member.conversationId}`
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const updated = {
|
||||
...omit(member, 'conversationId'),
|
||||
uuid,
|
||||
};
|
||||
|
||||
// We previously stored our conversation
|
||||
if (!('addedByUserId' in member) || !member.addedByUserId) {
|
||||
return updated;
|
||||
}
|
||||
|
||||
const addedByUserId:
|
||||
| UUIDStringType
|
||||
| undefined = getConversationUuid.get({
|
||||
conversationId: member.addedByUserId,
|
||||
});
|
||||
|
||||
if (!addedByUserId) {
|
||||
return updated;
|
||||
}
|
||||
|
||||
addedByCount += 1;
|
||||
|
||||
return {
|
||||
...updated,
|
||||
addedByUserId,
|
||||
};
|
||||
})
|
||||
.filter(isNotNil);
|
||||
|
||||
result = {
|
||||
...result,
|
||||
[key]: newValue,
|
||||
};
|
||||
|
||||
if (oldValue.length !== 0) {
|
||||
logger.info(
|
||||
`updateToSchemaVersion43: migrated ${oldValue.length} ${key} ` +
|
||||
`entries to ${newValue.length} for ${legacy.id}`
|
||||
);
|
||||
}
|
||||
|
||||
if (addedByCount > 0) {
|
||||
logger.info(
|
||||
`updateToSchemaVersion43: migrated ${addedByCount} addedByUserId ` +
|
||||
`in ${key} for ${legacy.id}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (result === convo) {
|
||||
return;
|
||||
}
|
||||
|
||||
let dbMembers: string | null;
|
||||
if (result.membersV2) {
|
||||
dbMembers = result.membersV2.map(item => item.uuid).join(' ');
|
||||
} else if (result.members) {
|
||||
dbMembers = result.members.join(' ');
|
||||
} else {
|
||||
dbMembers = null;
|
||||
}
|
||||
|
||||
updateConversationStmt.run({
|
||||
id: result.id,
|
||||
json: objectToJSON(result),
|
||||
members: dbMembers,
|
||||
});
|
||||
};
|
||||
|
||||
type LegacyMessageType = {
|
||||
id: string;
|
||||
groupV2Change?: {
|
||||
from: string;
|
||||
details: Array<
|
||||
(
|
||||
| {
|
||||
type:
|
||||
| 'member-add'
|
||||
| 'member-add-from-invite'
|
||||
| 'member-add-from-link'
|
||||
| 'member-add-from-admin-approval'
|
||||
| 'member-privilege'
|
||||
| 'member-remove'
|
||||
| 'pending-add-one'
|
||||
| 'pending-remove-one'
|
||||
| 'admin-approval-add-one'
|
||||
| 'admin-approval-remove-one';
|
||||
conversationId: string;
|
||||
}
|
||||
| {
|
||||
type: unknown;
|
||||
conversationId?: undefined;
|
||||
}
|
||||
) &
|
||||
(
|
||||
| {
|
||||
type:
|
||||
| 'member-add-from-invite'
|
||||
| 'pending-remove-one'
|
||||
| 'pending-remove-many'
|
||||
| 'admin-approval-remove-one';
|
||||
inviter: string;
|
||||
}
|
||||
| {
|
||||
inviter?: undefined;
|
||||
}
|
||||
)
|
||||
>;
|
||||
};
|
||||
sourceUuid: string;
|
||||
invitedGV2Members?: Array<LegacyPendingMemberType>;
|
||||
};
|
||||
|
||||
const upgradeMessage = (message: MessageType): boolean => {
|
||||
const {
|
||||
id,
|
||||
groupV2Change,
|
||||
sourceUuid,
|
||||
invitedGV2Members,
|
||||
} = (message as unknown) as LegacyMessageType;
|
||||
let result = message;
|
||||
|
||||
if (groupV2Change) {
|
||||
assert(result.groupV2Change, 'Pacify typescript');
|
||||
|
||||
const from: UUIDStringType | undefined = getConversationUuid.get({
|
||||
conversationId: groupV2Change.from,
|
||||
});
|
||||
|
||||
if (from) {
|
||||
result = {
|
||||
...result,
|
||||
groupV2Change: {
|
||||
...result.groupV2Change,
|
||||
from,
|
||||
},
|
||||
};
|
||||
} else {
|
||||
result = {
|
||||
...result,
|
||||
groupV2Change: omit(result.groupV2Change, ['from']),
|
||||
};
|
||||
}
|
||||
|
||||
let changedDetails = false;
|
||||
const details = groupV2Change.details
|
||||
.map((legacyDetail, i) => {
|
||||
const oldDetail = result.groupV2Change?.details[i];
|
||||
assert(oldDetail, 'Pacify typescript');
|
||||
let newDetail = oldDetail;
|
||||
|
||||
for (const key of ['conversationId' as const, 'inviter' as const]) {
|
||||
const oldValue = legacyDetail[key];
|
||||
const newKey = key === 'conversationId' ? 'uuid' : key;
|
||||
|
||||
if (oldValue === undefined) {
|
||||
continue;
|
||||
}
|
||||
changedDetails = true;
|
||||
|
||||
let newValue: UUIDStringType | null = getConversationUuid.get({
|
||||
conversationId: oldValue,
|
||||
});
|
||||
if (key === 'inviter') {
|
||||
newValue = newValue ?? UUID.cast(oldValue);
|
||||
}
|
||||
if (!newValue) {
|
||||
logger.warn(
|
||||
`updateToSchemaVersion43: ${id}.groupV2Change.details.${key} ` +
|
||||
`UUID not found for ${oldValue}`
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
assert(newDetail.type === legacyDetail.type, 'Pacify typescript');
|
||||
newDetail = {
|
||||
...omit(newDetail, key),
|
||||
[newKey]: newValue,
|
||||
};
|
||||
}
|
||||
|
||||
return newDetail;
|
||||
})
|
||||
.filter(isNotNil);
|
||||
|
||||
if (changedDetails) {
|
||||
result = {
|
||||
...result,
|
||||
groupV2Change: {
|
||||
...result.groupV2Change,
|
||||
details,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceUuid) {
|
||||
const newValue: UUIDStringType =
|
||||
getConversationUuid.get({
|
||||
conversationId: sourceUuid,
|
||||
}) ?? UUID.cast(sourceUuid);
|
||||
|
||||
if (newValue !== sourceUuid) {
|
||||
result = {
|
||||
...result,
|
||||
sourceUuid: newValue,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (invitedGV2Members) {
|
||||
const newMembers = invitedGV2Members
|
||||
.map(({ addedByUserId, conversationId }, i) => {
|
||||
const uuid: UUIDStringType | null = getConversationUuid.get({
|
||||
conversationId,
|
||||
});
|
||||
const oldMember =
|
||||
result.invitedGV2Members && result.invitedGV2Members[i];
|
||||
assert(oldMember !== undefined, 'Pacify typescript');
|
||||
|
||||
if (!uuid) {
|
||||
logger.warn(
|
||||
`updateToSchemaVersion43: ${id}.invitedGV2Members UUID ` +
|
||||
`not found for ${conversationId}`
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const newMember = {
|
||||
...omit(oldMember, ['conversationId']),
|
||||
uuid,
|
||||
};
|
||||
|
||||
if (!addedByUserId) {
|
||||
return newMember;
|
||||
}
|
||||
|
||||
const newAddedBy: UUIDStringType | null = getConversationUuid.get({
|
||||
conversationId: addedByUserId,
|
||||
});
|
||||
if (!newAddedBy) {
|
||||
return newMember;
|
||||
}
|
||||
|
||||
return {
|
||||
...newMember,
|
||||
addedByUserId: newAddedBy,
|
||||
};
|
||||
})
|
||||
.filter(isNotNil);
|
||||
|
||||
result = {
|
||||
...result,
|
||||
invitedGV2Members: newMembers,
|
||||
};
|
||||
}
|
||||
|
||||
if (result === message) {
|
||||
return false;
|
||||
}
|
||||
|
||||
updateMessageStmt.run({
|
||||
id: result.id,
|
||||
json: JSON.stringify(result),
|
||||
sourceUuid: result.sourceUuid ?? null,
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
db.transaction(() => {
|
||||
const allConversations = db
|
||||
.prepare<EmptyQuery>(
|
||||
`
|
||||
SELECT json, profileLastFetchedAt
|
||||
FROM conversations
|
||||
ORDER BY id ASC;
|
||||
`
|
||||
)
|
||||
.all()
|
||||
.map(({ json }) => jsonToObject<ConversationType>(json));
|
||||
|
||||
logger.info(
|
||||
'updateToSchemaVersion43: About to iterate through ' +
|
||||
`${allConversations.length} conversations`
|
||||
);
|
||||
|
||||
for (const convo of allConversations) {
|
||||
upgradeConversation(convo);
|
||||
}
|
||||
|
||||
const messageCount = getCountFromTable(db, 'messages');
|
||||
logger.info(
|
||||
'updateToSchemaVersion43: About to iterate through ' +
|
||||
`${messageCount} messages`
|
||||
);
|
||||
|
||||
let updatedCount = 0;
|
||||
for (const message of new TableIterator<MessageType>(db, 'messages')) {
|
||||
if (upgradeMessage(message)) {
|
||||
updatedCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`updateToSchemaVersion43: Updated ${updatedCount} messages`);
|
||||
|
||||
db.pragma('user_version = 43');
|
||||
})();
|
||||
logger.info('updateToSchemaVersion43: success!');
|
||||
}
|
||||
1934
ts/sql/migrations/index.ts
Normal file
1934
ts/sql/migrations/index.ts
Normal file
File diff suppressed because it is too large
Load diff
260
ts/sql/util.ts
Normal file
260
ts/sql/util.ts
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
// Copyright 2021 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import type { Database } from 'better-sqlite3';
|
||||
import { isNumber, last } from 'lodash';
|
||||
|
||||
export type EmptyQuery = [];
|
||||
export type ArrayQuery = Array<Array<null | number | bigint | string>>;
|
||||
export type Query = { [key: string]: null | number | bigint | string | Buffer };
|
||||
export type JSONRows = Array<{ readonly json: string }>;
|
||||
|
||||
export type TableType =
|
||||
| 'attachment_downloads'
|
||||
| 'conversations'
|
||||
| 'identityKeys'
|
||||
| 'items'
|
||||
| 'messages'
|
||||
| 'preKeys'
|
||||
| 'senderKeys'
|
||||
| 'sessions'
|
||||
| 'signedPreKeys'
|
||||
| 'stickers'
|
||||
| 'unprocessed';
|
||||
|
||||
// This value needs to be below SQLITE_MAX_VARIABLE_NUMBER.
|
||||
const MAX_VARIABLE_COUNT = 100;
|
||||
|
||||
export function objectToJSON<T>(data: T): string {
|
||||
return JSON.stringify(data);
|
||||
}
|
||||
|
||||
export function jsonToObject<T>(json: string): T {
|
||||
return JSON.parse(json);
|
||||
}
|
||||
|
||||
//
|
||||
// Database helpers
|
||||
//
|
||||
|
||||
export function getSQLiteVersion(db: Database): string {
|
||||
const { sqlite_version: version } = db
|
||||
.prepare<EmptyQuery>('select sqlite_version() AS sqlite_version')
|
||||
.get();
|
||||
|
||||
return version;
|
||||
}
|
||||
|
||||
export function getSchemaVersion(db: Database): number {
|
||||
return db.pragma('schema_version', { simple: true });
|
||||
}
|
||||
|
||||
export function setUserVersion(db: Database, version: number): void {
|
||||
if (!isNumber(version)) {
|
||||
throw new Error(`setUserVersion: version ${version} is not a number`);
|
||||
}
|
||||
db.pragma(`user_version = ${version}`);
|
||||
}
|
||||
|
||||
export function getUserVersion(db: Database): number {
|
||||
return db.pragma('user_version', { simple: true });
|
||||
}
|
||||
|
||||
export function getSQLCipherVersion(db: Database): string | undefined {
|
||||
return db.pragma('cipher_version', { simple: true });
|
||||
}
|
||||
|
||||
//
|
||||
// Various table helpers
|
||||
//
|
||||
|
||||
export function batchMultiVarQuery<ValueT>(
|
||||
db: Database,
|
||||
values: Array<ValueT>,
|
||||
query: (batch: Array<ValueT>) => void
|
||||
): [];
|
||||
export function batchMultiVarQuery<ValueT, ResultT>(
|
||||
db: Database,
|
||||
values: Array<ValueT>,
|
||||
query: (batch: Array<ValueT>) => Array<ResultT>
|
||||
): Array<ResultT>;
|
||||
|
||||
export function batchMultiVarQuery<ValueT, ResultT>(
|
||||
db: Database,
|
||||
values: Array<ValueT>,
|
||||
query:
|
||||
| ((batch: Array<ValueT>) => void)
|
||||
| ((batch: Array<ValueT>) => Array<ResultT>)
|
||||
): Array<ResultT> {
|
||||
if (values.length > MAX_VARIABLE_COUNT) {
|
||||
const result: Array<ResultT> = [];
|
||||
db.transaction(() => {
|
||||
for (let i = 0; i < values.length; i += MAX_VARIABLE_COUNT) {
|
||||
const batch = values.slice(i, i + MAX_VARIABLE_COUNT);
|
||||
const batchResult = query(batch);
|
||||
if (Array.isArray(batchResult)) {
|
||||
result.push(...batchResult);
|
||||
}
|
||||
}
|
||||
})();
|
||||
return result;
|
||||
}
|
||||
|
||||
const result = query(values);
|
||||
return Array.isArray(result) ? result : [];
|
||||
}
|
||||
|
||||
export function createOrUpdate<Key extends string | number>(
|
||||
db: Database,
|
||||
table: TableType,
|
||||
data: Record<string, unknown> & { id: Key }
|
||||
): void {
|
||||
const { id } = data;
|
||||
if (!id) {
|
||||
throw new Error('createOrUpdate: Provided data did not have a truthy id');
|
||||
}
|
||||
|
||||
db.prepare<Query>(
|
||||
`
|
||||
INSERT OR REPLACE INTO ${table} (
|
||||
id,
|
||||
json
|
||||
) values (
|
||||
$id,
|
||||
$json
|
||||
)
|
||||
`
|
||||
).run({
|
||||
id,
|
||||
json: objectToJSON(data),
|
||||
});
|
||||
}
|
||||
|
||||
export function bulkAdd(
|
||||
db: Database,
|
||||
table: TableType,
|
||||
array: Array<Record<string, unknown> & { id: string | number }>
|
||||
): void {
|
||||
db.transaction(() => {
|
||||
for (const data of array) {
|
||||
createOrUpdate(db, table, data);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
export function getById<Key extends string | number, Result = unknown>(
|
||||
db: Database,
|
||||
table: TableType,
|
||||
id: Key
|
||||
): Result | undefined {
|
||||
const row = db
|
||||
.prepare<Query>(
|
||||
`
|
||||
SELECT *
|
||||
FROM ${table}
|
||||
WHERE id = $id;
|
||||
`
|
||||
)
|
||||
.get({
|
||||
id,
|
||||
});
|
||||
|
||||
if (!row) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return jsonToObject(row.json);
|
||||
}
|
||||
|
||||
export function removeById<Key extends string | number>(
|
||||
db: Database,
|
||||
table: TableType,
|
||||
id: Key | Array<Key>
|
||||
): void {
|
||||
if (!Array.isArray(id)) {
|
||||
db.prepare<Query>(
|
||||
`
|
||||
DELETE FROM ${table}
|
||||
WHERE id = $id;
|
||||
`
|
||||
).run({ id });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!id.length) {
|
||||
throw new Error('removeById: No ids to delete!');
|
||||
}
|
||||
|
||||
const removeByIdsSync = (ids: Array<string | number>): void => {
|
||||
db.prepare<ArrayQuery>(
|
||||
`
|
||||
DELETE FROM ${table}
|
||||
WHERE id IN ( ${id.map(() => '?').join(', ')} );
|
||||
`
|
||||
).run(ids);
|
||||
};
|
||||
|
||||
batchMultiVarQuery(db, id, removeByIdsSync);
|
||||
}
|
||||
|
||||
export function removeAllFromTable(db: Database, table: TableType): void {
|
||||
db.prepare<EmptyQuery>(`DELETE FROM ${table};`).run();
|
||||
}
|
||||
|
||||
export function getAllFromTable<T>(db: Database, table: TableType): Array<T> {
|
||||
const rows: JSONRows = db
|
||||
.prepare<EmptyQuery>(`SELECT json FROM ${table};`)
|
||||
.all();
|
||||
|
||||
return rows.map(row => jsonToObject(row.json));
|
||||
}
|
||||
|
||||
export function getCountFromTable(db: Database, table: TableType): number {
|
||||
const result: null | number = db
|
||||
.prepare<EmptyQuery>(`SELECT count(*) from ${table};`)
|
||||
.pluck(true)
|
||||
.get();
|
||||
if (isNumber(result)) {
|
||||
return result;
|
||||
}
|
||||
throw new Error(`getCountFromTable: Unable to get count from table ${table}`);
|
||||
}
|
||||
|
||||
export class TableIterator<ObjectType extends { id: string }> {
|
||||
constructor(
|
||||
private readonly db: Database,
|
||||
private readonly table: TableType,
|
||||
private readonly pageSize = 500
|
||||
) {}
|
||||
|
||||
*[Symbol.iterator](): Iterator<ObjectType> {
|
||||
const fetchObject = this.db.prepare<Query>(
|
||||
`
|
||||
SELECT json FROM ${this.table}
|
||||
WHERE id > $id
|
||||
ORDER BY id ASC
|
||||
LIMIT $pageSize;
|
||||
`
|
||||
);
|
||||
|
||||
let complete = false;
|
||||
let id = '';
|
||||
while (!complete) {
|
||||
const rows: JSONRows = fetchObject.all({
|
||||
id,
|
||||
pageSize: this.pageSize,
|
||||
});
|
||||
|
||||
const messages: Array<ObjectType> = rows.map(row =>
|
||||
jsonToObject(row.json)
|
||||
);
|
||||
yield* messages;
|
||||
|
||||
const lastMessage: ObjectType | undefined = last(messages);
|
||||
if (lastMessage) {
|
||||
({ id } = lastMessage);
|
||||
}
|
||||
complete = messages.length < this.pageSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue