mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Migration and data access functions for stories

Browse source
This commit is contained in:
Scott Nonnenberg 2021-12-08 11:52:46 -08:00 • committed by GitHub
parent 9f4a01c535
commit fdc9885baa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 3428 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

156
ts/sql/Client.ts
View file

@ -80,6 +80,10 @@ import type {
StickerPackStatusType,
StickerPackType,
StickerType,
StoryDistributionType,
StoryDistributionMemberType,
StoryDistributionWithMembersType,
StoryReadType,
UnprocessedType,
UnprocessedUpdateType,
} from './Interface';
@ -217,13 +221,14 @@ const dataInterface: ClientInterface = {
saveMessages,
removeMessage,
removeMessages,
getUnreadCountForConversation,
getTotalUnreadForConversation,
getUnreadByConversationAndMarkRead,
getUnreadReactionsAndMarkRead,
markReactionAsRead,
removeReactionFromConversation,
addReaction,
_getAllReactions,
_removeAllReactions,
getMessageBySender,
getMessageById,
@ -236,6 +241,7 @@ const dataInterface: ClientInterface = {
getNextTapToViewMessageTimestampToAgeOut,
getTapToViewMessagesNeedingErase,
getOlderMessagesByConversation,
getOlderStories,
getNewerMessagesByConversation,
getLastConversationMessages,
getMessageMetricsForConversation,
@ -277,6 +283,19 @@ const dataInterface: ClientInterface = {
updateOrCreateBadges,
badgeImageFileDownloaded,
_getAllStoryDistributions,
_getAllStoryDistributionMembers,
_deleteAllStoryDistributions,
createNewStoryDistribution,
getAllStoryDistributionsWithMembers,
modifyStoryDistributionMembers,
deleteStoryDistribution,
_getAllStoryReads,
_deleteAllStoryReads,
addNewStoryRead,
getLastStoryReadsForAuthor,
removeAll,
removeAllConfiguration,
@ -300,6 +319,7 @@ const dataInterface: ClientInterface = {
// Test-only
_getAllMessages,
_removeAllMessages,
// Client-side only
@ -1190,6 +1210,9 @@ async function _getAllMessages({
return new MessageCollection(messages);
}
async function _removeAllMessages() {
await channels._removeAllMessages();
}
async function getAllMessageIds() {
const ids = await channels.getAllMessageIds();
@ -1224,27 +1247,28 @@ async function getMessageBySender(
return new Message(messages[0]);
}
async function getUnreadCountForConversation(conversationId: string) {
return channels.getUnreadCountForConversation(conversationId);
async function getTotalUnreadForConversation(
conversationId: string,
storyId?: UUIDStringType
) {
return channels.getTotalUnreadForConversation(conversationId, storyId);
}
async function getUnreadByConversationAndMarkRead(
conversationId: string,
newestUnreadId: number,
readAt?: number
) {
return channels.getUnreadByConversationAndMarkRead(
conversationId,
newestUnreadId,
readAt
);
async function getUnreadByConversationAndMarkRead(options: {
conversationId: string;
newestUnreadAt: number;
readAt?: number;
storyId?: UUIDStringType;
}) {
return channels.getUnreadByConversationAndMarkRead(options);
}
async function getUnreadReactionsAndMarkRead(
conversationId: string,
newestUnreadId: number
) {
return channels.getUnreadReactionsAndMarkRead(conversationId, newestUnreadId);
async function getUnreadReactionsAndMarkRead(options: {
conversationId: string;
newestUnreadAt: number;
storyId?: UUIDStringType;
}) {
return channels.getUnreadReactionsAndMarkRead(options);
}
async function markReactionAsRead(
@ -1270,6 +1294,9 @@ async function addReaction(reactionObj: ReactionType) {
async function _getAllReactions() {
return channels._getAllReactions();
}
async function _removeAllReactions() {
await channels._removeAllReactions();
}
function handleMessageJSON(messages: Array<MessageTypeUnhydrated>) {
return messages.map(message => JSON.parse(message.json));
@ -1279,16 +1306,18 @@ async function getOlderMessagesByConversation(
conversationId: string,
{
limit = 100,
MessageCollection,
messageId,
receivedAt = Number.MAX_VALUE,
sentAt = Number.MAX_VALUE,
messageId,
MessageCollection,
storyId,
}: {
limit?: number;
MessageCollection: typeof MessageModelCollectionType;
messageId?: string;
receivedAt?: number;
sentAt?: number;
messageId?: string;
MessageCollection: typeof MessageModelCollectionType;
storyId?: UUIDStringType;
}
) {
const messages = await channels.getOlderMessagesByConversation(
@ -1298,23 +1327,36 @@ async function getOlderMessagesByConversation(
receivedAt,
sentAt,
messageId,
storyId,
}
);
return new MessageCollection(handleMessageJSON(messages));
}
async function getOlderStories(options: {
conversationId?: string;
limit?: number;
receivedAt?: number;
sentAt?: number;
sourceUuid?: string;
}): Promise<Array<MessageType>> {
return channels.getOlderStories(options);
}
async function getNewerMessagesByConversation(
conversationId: string,
{
limit = 100,
MessageCollection,
receivedAt = 0,
sentAt = 0,
MessageCollection,
storyId,
}: {
limit?: number;
MessageCollection: typeof MessageModelCollectionType;
receivedAt?: number;
sentAt?: number;
MessageCollection: typeof MessageModelCollectionType;
storyId?: UUIDStringType;
}
) {
const messages = await channels.getNewerMessagesByConversation(
@ -1323,6 +1365,7 @@ async function getNewerMessagesByConversation(
limit,
receivedAt,
sentAt,
storyId,
}
);
@ -1349,9 +1392,13 @@ async function getLastConversationMessages({
hasUserInitiatedMessages,
};
}
async function getMessageMetricsForConversation(conversationId: string) {
async function getMessageMetricsForConversation(
conversationId: string,
storyId?: UUIDStringType
) {
const result = await channels.getMessageMetricsForConversation(
conversationId
conversationId,
storyId
);
return result;
@ -1597,6 +1644,63 @@ function badgeImageFileDownloaded(
return channels.badgeImageFileDownloaded(url, localPath);
}
// Story Distributions
async function _getAllStoryDistributions(): Promise<
Array<StoryDistributionType>
> {
return channels._getAllStoryDistributions();
}
async function _getAllStoryDistributionMembers(): Promise<
Array<StoryDistributionMemberType>
> {
return channels._getAllStoryDistributionMembers();
}
async function _deleteAllStoryDistributions(): Promise<void> {
await channels._deleteAllStoryDistributions();
}
async function createNewStoryDistribution(
story: StoryDistributionWithMembersType
): Promise<void> {
await channels.createNewStoryDistribution(story);
}
async function getAllStoryDistributionsWithMembers(): Promise<
Array<StoryDistributionWithMembersType>
> {
return channels.getAllStoryDistributionsWithMembers();
}
async function modifyStoryDistributionMembers(
id: string,
options: {
toAdd: Array<UUIDStringType>;
toRemove: Array<UUIDStringType>;
}
): Promise<void> {
await channels.modifyStoryDistributionMembers(id, options);
}
async function deleteStoryDistribution(id: UUIDStringType): Promise<void> {
await channels.deleteStoryDistribution(id);
}
// Story Reads
async function _getAllStoryReads(): Promise<Array<StoryReadType>> {
return channels._getAllStoryReads();
}
async function _deleteAllStoryReads(): Promise<void> {
await channels._deleteAllStoryReads();
}
async function addNewStoryRead(read: StoryReadType): Promise<void> {
return channels.addNewStoryRead(read);
}
async function getLastStoryReadsForAuthor(options: {
authorId: UUIDStringType;
conversationId?: UUIDStringType;
limit?: number;
}): Promise<Array<StoryReadType>> {
return channels.getLastStoryReadsForAuthor(options);
}
// Other
async function removeAll() {

106
ts/sql/Interface.ts
View file

@ -9,6 +9,7 @@ import type {
ConversationModelCollectionType,
MessageAttributesType,
MessageModelCollectionType,
SenderKeyInfoType,
} from '../model-types.d';
import type { MessageModel } from '../models/messages';
import type { ConversationModel } from '../models/conversations';
@ -239,6 +240,31 @@ export type DeleteSentProtoRecipientOptionsType = Readonly<{
deviceId: number;
}>;
export type StoryDistributionType = Readonly<{
id: UUIDStringType;
name: string;
avatarUrlPath: string;
avatarKey: Uint8Array;
senderKeyInfo: SenderKeyInfoType | undefined;
}>;
export type StoryDistributionMemberType = Readonly<{
listId: UUIDStringType;
uuid: UUIDStringType;
}>;
export type StoryDistributionWithMembersType = Readonly<
{
members: Array<UUIDStringType>;
} & StoryDistributionType
>;
export type StoryReadType = Readonly<{
authorId: UUIDStringType;
conversationId: UUIDStringType;
storyId: UUIDStringType;
storyReadDate: number;
}>;
export type DataInterface = {
close: () => Promise<void>;
removeDB: () => Promise<void>;
@ -349,8 +375,16 @@ export type DataInterface = {
) => Promise<void>;
getMessageCount: (conversationId?: string) => Promise<number>;
getAllMessageIds: () => Promise<Array<string>>;
getOlderStories: (options: {
conversationId?: string;
limit?: number;
receivedAt?: number;
sentAt?: number;
sourceUuid?: string;
}) => Promise<Array<MessageType>>;
getMessageMetricsForConversation: (
conversationId: string
conversationId: string,
storyId?: UUIDStringType
) => Promise<ConversationMetricsType>;
hasGroupCallHistoryMessage: (
conversationId: string,
@ -361,21 +395,27 @@ export type DataInterface = {
currentId: string
) => Promise<void>;
getNextTapToViewMessageTimestampToAgeOut: () => Promise<undefined | number>;
_removeAllMessages: () => Promise<void>;
getUnreadCountForConversation: (conversationId: string) => Promise<number>;
getUnreadByConversationAndMarkRead: (
getTotalUnreadForConversation: (
conversationId: string,
newestUnreadId: number,
readAt?: number
) => Promise<
storyId?: UUIDStringType
) => Promise<number>;
getUnreadByConversationAndMarkRead: (options: {
conversationId: string;
newestUnreadAt: number;
readAt?: number;
storyId?: UUIDStringType;
}) => Promise<
Array<
Pick<MessageType, 'id' | 'source' | 'sourceUuid' | 'sent_at' | 'type'>
>
>;
getUnreadReactionsAndMarkRead: (
conversationId: string,
newestUnreadId: number
) => Promise<
getUnreadReactionsAndMarkRead: (options: {
conversationId: string;
newestUnreadAt: number;
storyId?: UUIDStringType;
}) => Promise<
Array<
Pick<ReactionType, 'targetAuthorUuid' | 'targetTimestamp' | 'messageId'>
>
@ -392,6 +432,7 @@ export type DataInterface = {
}) => Promise<void>;
addReaction: (reactionObj: ReactionType) => Promise<void>;
_getAllReactions: () => Promise<Array<ReactionType>>;
_removeAllReactions: () => Promise<void>;
getUnprocessedCount: () => Promise<number>;
getAllUnprocessed: () => Promise<Array<UnprocessedType>>;
@ -452,6 +493,35 @@ export type DataInterface = {
updateOrCreateBadges(badges: ReadonlyArray<BadgeType>): Promise<void>;
badgeImageFileDownloaded(url: string, localPath: string): Promise<void>;
_getAllStoryDistributions(): Promise<Array<StoryDistributionType>>;
_getAllStoryDistributionMembers(): Promise<
Array<StoryDistributionMemberType>
>;
_deleteAllStoryDistributions(): Promise<void>;
createNewStoryDistribution(
story: StoryDistributionWithMembersType
): Promise<void>;
getAllStoryDistributionsWithMembers(): Promise<
Array<StoryDistributionWithMembersType>
>;
modifyStoryDistributionMembers(
id: string,
options: {
toAdd: Array<UUIDStringType>;
toRemove: Array<UUIDStringType>;
}
): Promise<void>;
deleteStoryDistribution(id: UUIDStringType): Promise<void>;
_getAllStoryReads(): Promise<Array<StoryReadType>>;
_deleteAllStoryReads(): Promise<void>;
addNewStoryRead(read: StoryReadType): Promise<void>;
getLastStoryReadsForAuthor(options: {
authorId: UUIDStringType;
conversationId?: UUIDStringType;
limit?: number;
}): Promise<Array<StoryReadType>>;
removeAll: () => Promise<void>;
removeAllConfiguration: (type?: RemoveAllConfiguration) => Promise<void>;
@ -525,14 +595,20 @@ export type ServerInterface = DataInterface & {
conversationId: string,
options?: {
limit?: number;
messageId?: string;
receivedAt?: number;
sentAt?: number;
messageId?: string;
storyId?: UUIDStringType;
}
) => Promise<Array<MessageTypeUnhydrated>>;
getNewerMessagesByConversation: (
conversationId: string,
options?: { limit?: number; receivedAt?: number; sentAt?: number }
options?: {
limit?: number;
receivedAt?: number;
sentAt?: number;
storyId?: UUIDStringType;
}
) => Promise<Array<MessageTypeUnhydrated>>;
getLastConversationMessages: (options: {
conversationId: string;
@ -622,19 +698,21 @@ export type ClientInterface = DataInterface & {
conversationId: string,
options: {
limit?: number;
MessageCollection: typeof MessageModelCollectionType;
messageId?: string;
receivedAt?: number;
sentAt?: number;
MessageCollection: typeof MessageModelCollectionType;
storyId?: UUIDStringType;
}
) => Promise<MessageModelCollectionType>;
getNewerMessagesByConversation: (
conversationId: string,
options: {
limit?: number;
MessageCollection: typeof MessageModelCollectionType;
receivedAt?: number;
sentAt?: number;
MessageCollection: typeof MessageModelCollectionType;
storyId?: UUIDStringType;
}
) => Promise<MessageModelCollectionType>;
getLastConversationMessages: (options: {

612
ts/sql/Server.ts
View file

@ -14,6 +14,7 @@ import type { Dictionary } from 'lodash';
import {
forEach,
fromPairs,
groupBy,
isNil,
isNumber,
isString,
@ -75,19 +76,19 @@ import type {
ConversationType,
DeleteSentProtoRecipientOptionsType,
EmojiType,
IdentityKeyType,
IdentityKeyIdType,
IdentityKeyType,
ItemKeyType,
ItemType,
LastConversationMessagesServerType,
MessageMetricsType,
MessageType,
MessageTypeUnhydrated,
PreKeyType,
PreKeyIdType,
PreKeyType,
SearchResultMessageType,
SenderKeyType,
SenderKeyIdType,
SenderKeyType,
SentMessageDBType,
SentMessagesType,
SentProtoType,
@ -95,13 +96,17 @@ import type {
SentRecipientsDBType,
SentRecipientsType,
ServerInterface,
SessionType,
SessionIdType,
SignedPreKeyType,
SessionType,
SignedPreKeyIdType,
SignedPreKeyType,
StickerPackStatusType,
StickerPackType,
StickerType,
StoryDistributionMemberType,
StoryDistributionType,
StoryDistributionWithMembersType,
StoryReadType,
UnprocessedType,
UnprocessedUpdateType,
} from './Interface';
@ -207,17 +212,18 @@ const dataInterface: ServerInterface = {
saveMessages,
removeMessage,
removeMessages,
getUnreadCountForConversation,
getUnreadByConversationAndMarkRead,
getUnreadReactionsAndMarkRead,
markReactionAsRead,
addReaction,
removeReactionFromConversation,
_getAllReactions,
_removeAllReactions,
getMessageBySender,
getMessageById,
getMessagesById,
_getAllMessages,
_removeAllMessages,
getAllMessageIds,
getMessagesBySentAt,
getExpiredMessages,
@ -226,7 +232,9 @@ const dataInterface: ServerInterface = {
getNextTapToViewMessageTimestampToAgeOut,
getTapToViewMessagesNeedingErase,
getOlderMessagesByConversation,
getOlderStories,
getNewerMessagesByConversation,
getTotalUnreadForConversation,
getMessageMetricsForConversation,
getLastConversationMessages,
hasGroupCallHistoryMessage,
@ -267,6 +275,19 @@ const dataInterface: ServerInterface = {
updateOrCreateBadges,
badgeImageFileDownloaded,
_getAllStoryDistributions,
_getAllStoryDistributionMembers,
_deleteAllStoryDistributions,
createNewStoryDistribution,
getAllStoryDistributionsWithMembers,
modifyStoryDistributionMembers,
deleteStoryDistribution,
_getAllStoryReads,
_deleteAllStoryReads,
addNewStoryRead,
getLastStoryReadsForAuthor,
removeAll,
removeAllConfiguration,
@ -1674,8 +1695,6 @@ async function getMessageCount(conversationId?: string): Promise<number> {
function hasUserInitiatedMessages(conversationId: string): boolean {
const db = getInstance();
// We apply the limit in the sub-query so that `json_extract` wouldn't run
// for additional messages.
const row: { count: number } = db
.prepare<Query>(
`
@ -1687,14 +1706,15 @@ function hasUserInitiatedMessages(conversationId: string): boolean {
(type IS NULL
OR
type NOT IN (
'profile-change',
'verified-change',
'message-history-unsynced',
'keychange',
'group-v1-migration',
'universal-timer-notification',
'change-number-notification',
'group-v2-change'
'group-v1-migration',
'group-v2-change',
'keychange',
'message-history-unsynced',
'profile-change',
'story',
'universal-timer-notification',
'verified-change'
)
)
LIMIT 1
@ -1749,6 +1769,7 @@ function saveMessageSync(
source,
sourceUuid,
sourceDevice,
storyId,
type,
readStatus,
expireTimer,
@ -1775,6 +1796,7 @@ function saveMessageSync(
source: source || null,
sourceUuid: sourceUuid || null,
sourceDevice: sourceDevice || null,
storyId: storyId || null,
type: type || null,
readStatus: readStatus ?? null,
};
@ -1803,6 +1825,7 @@ function saveMessageSync(
source = $source,
sourceUuid = $sourceUuid,
sourceDevice = $sourceDevice,
storyId = $storyId,
type = $type,
readStatus = $readStatus
WHERE id = $id;
@ -1844,6 +1867,7 @@ function saveMessageSync(
source,
sourceUuid,
sourceDevice,
storyId,
type,
readStatus
) values (
@ -1866,6 +1890,7 @@ function saveMessageSync(
$source,
$sourceUuid,
$sourceDevice,
$storyId,
$type,
$readStatus
);
@ -1974,6 +1999,10 @@ async function _getAllMessages(): Promise<Array<MessageType>> {
return rows.map(row => jsonToObject(row.json));
}
async function _removeAllMessages(): Promise<void> {
const db = getInstance();
db.prepare<EmptyQuery>('DELETE from messages;').run();
}
async function getAllMessageIds(): Promise<Array<string>> {
const db = getInstance();
@ -2014,30 +2043,17 @@ async function getMessageBySender({
return rows.map(row => jsonToObject(row.json));
}
async function getUnreadCountForConversation(
conversationId: string
): Promise<number> {
const db = getInstance();
const row = db
.prepare<Query>(
`
SELECT COUNT(*) AS unreadCount FROM messages
WHERE readStatus = ${ReadStatus.Unread} AND
conversationId = $conversationId AND
type = 'incoming';
`
)
.get({
conversationId,
});
return row.unreadCount;
}
async function getUnreadByConversationAndMarkRead(
conversationId: string,
newestUnreadId: number,
readAt?: number
): Promise<
async function getUnreadByConversationAndMarkRead({
conversationId,
newestUnreadAt,
storyId,
readAt,
}: {
conversationId: string;
newestUnreadAt: number;
storyId?: UUIDStringType;
readAt?: number;
}): Promise<
Array<Pick<MessageType, 'id' | 'source' | 'sourceUuid' | 'sent_at' | 'type'>>
> {
const db = getInstance();
@ -2057,13 +2073,15 @@ async function getUnreadByConversationAndMarkRead(
) AND
expireTimer IS NOT NULL AND
conversationId = $conversationId AND
received_at <= $newestUnreadId;
storyId IS $storyId AND
received_at <= $newestUnreadAt;
`
).run({
conversationId,
expirationStartTimestamp,
jsonPatch: JSON.stringify({ expirationStartTimestamp }),
newestUnreadId,
newestUnreadAt,
storyId: storyId || null,
});
const rows = db
@ -2074,13 +2092,15 @@ async function getUnreadByConversationAndMarkRead(
WHERE
readStatus = ${ReadStatus.Unread} AND
conversationId = $conversationId AND
received_at <= $newestUnreadId
storyId IS $storyId AND
received_at <= $newestUnreadAt
ORDER BY received_at DESC, sent_at DESC;
`
)
.all({
conversationId,
newestUnreadId,
newestUnreadAt,
storyId: storyId || null,
});
db.prepare<Query>(
@ -2092,12 +2112,14 @@ async function getUnreadByConversationAndMarkRead(
WHERE
readStatus = ${ReadStatus.Unread} AND
conversationId = $conversationId AND
received_at <= $newestUnreadId;
storyId IS $storyId AND
received_at <= $newestUnreadAt;
`
).run({
conversationId,
jsonPatch: JSON.stringify({ readStatus: ReadStatus.Read }),
newestUnreadId,
newestUnreadAt,
storyId: storyId || null,
});
return rows.map(row => {
@ -2117,42 +2139,50 @@ async function getUnreadByConversationAndMarkRead(
})();
}
async function getUnreadReactionsAndMarkRead(
conversationId: string,
newestUnreadId: number
): Promise<
Array<
Pick<ReactionType, 'targetAuthorUuid' | 'targetTimestamp' | 'messageId'>
>
> {
type ReactionResultType = Pick<
ReactionType,
'targetAuthorUuid' | 'targetTimestamp' | 'messageId'
> & { rowid: number };
async function getUnreadReactionsAndMarkRead({
conversationId,
newestUnreadAt,
storyId,
}: {
conversationId: string;
newestUnreadAt: number;
storyId?: UUIDStringType;
}): Promise<Array<ReactionResultType>> {
const db = getInstance();
return db.transaction(() => {
const unreadMessages = db
const unreadMessages: Array<ReactionResultType> = db
.prepare<Query>(
`
SELECT targetAuthorUuid, targetTimestamp, messageId
FROM reactions WHERE
unread = 1 AND
conversationId = $conversationId AND
messageReceivedAt <= $newestUnreadId;
SELECT rowid, targetAuthorUuid, targetTimestamp, messageId
FROM reactions
JOIN messages on messages.id IS reactions.messageId
WHERE
unread IS NOT 0 AND
messages.conversationId IS $conversationId AND
messages.received_at <= $newestUnreadAt AND
messages.storyId IS $storyId
ORDER BY messageReceivedAt DESC;
`
)
.all({
conversationId,
newestUnreadId,
newestUnreadAt,
storyId: storyId || null,
});
db.prepare(
`
UPDATE reactions SET
unread = 0 WHERE
conversationId = $conversationId AND
messageReceivedAt <= $newestUnreadId;
`
).run({
conversationId,
newestUnreadId,
const idsToUpdate = unreadMessages.map(item => item.rowid);
batchMultiVarQuery(db, idsToUpdate, (ids: Array<number>): void => {
db.prepare<ArrayQuery>(
`
UPDATE reactions SET
unread = 0 WHERE rowid IN ( ${ids.map(() => '?').join(', ')} );
`
).run(idsToUpdate);
});
return unreadMessages;
@ -2275,69 +2305,95 @@ async function _getAllReactions(): Promise<Array<ReactionType>> {
const db = getInstance();
return db.prepare<EmptyQuery>('SELECT * from reactions;').all();
}
async function _removeAllReactions(): Promise<void> {
const db = getInstance();
db.prepare<EmptyQuery>('DELETE from reactions;').run();
}
async function getOlderMessagesByConversation(
conversationId: string,
{
limit = 100,
messageId,
receivedAt = Number.MAX_VALUE,
sentAt = Number.MAX_VALUE,
messageId,
storyId,
}: {
limit?: number;
messageId?: string;
receivedAt?: number;
sentAt?: number;
messageId?: string;
storyId?: UUIDStringType;
} = {}
): Promise<Array<MessageTypeUnhydrated>> {
const db = getInstance();
let rows: JSONRows;
if (messageId) {
rows = db
.prepare<Query>(
`
SELECT json FROM messages WHERE
conversationId = $conversationId AND
id != $messageId AND
(
(received_at = $received_at AND sent_at < $sent_at) OR
received_at < $received_at
)
ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;
`
)
.all({
conversationId,
received_at: receivedAt,
sent_at: sentAt,
limit,
messageId,
});
} else {
rows = db
.prepare<Query>(
`
SELECT json FROM messages WHERE
return db
.prepare<Query>(
`
SELECT json FROM messages WHERE
conversationId = $conversationId AND
($messageId IS NULL OR id IS NOT $messageId) AND
type IS NOT 'story' AND
storyId IS $storyId AND
(
(received_at = $received_at AND sent_at < $sent_at) OR
received_at < $received_at
)
ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;
`
)
.all({
conversationId,
received_at: receivedAt,
sent_at: sentAt,
limit,
});
}
ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;
`
)
.all({
conversationId,
limit,
messageId: messageId || null,
received_at: receivedAt,
sent_at: sentAt,
storyId: storyId || null,
})
.reverse();
}
return rows.reverse();
async function getOlderStories({
conversationId,
limit = 10,
receivedAt = Number.MAX_VALUE,
sentAt,
sourceUuid,
}: {
conversationId?: string;
limit?: number;
receivedAt?: number;
sentAt?: number;
sourceUuid?: string;
}): Promise<Array<MessageType>> {
const db = getInstance();
const rows: JSONRows = db
.prepare<Query>(
`
SELECT json
FROM messages
WHERE
type IS 'story' AND
($conversationId IS NULL OR conversationId IS $conversationId) AND
($sourceUuid IS NULL OR sourceUuid IS $sourceUuid) AND
(received_at < $receivedAt
OR (received_at IS $receivedAt AND sent_at < $sentAt)
)
ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;
`
)
.all({
conversationId: conversationId || null,
receivedAt,
sentAt: sentAt || null,
sourceUuid: sourceUuid || null,
limit,
});
return rows.map(row => jsonToObject(row.json));
}
async function getNewerMessagesByConversation(
@ -2346,7 +2402,13 @@ async function getNewerMessagesByConversation(
limit = 100,
receivedAt = 0,
sentAt = 0,
}: { limit?: number; receivedAt?: number; sentAt?: number } = {}
storyId,
}: {
limit?: number;
receivedAt?: number;
sentAt?: number;
storyId?: UUIDStringType;
} = {}
): Promise<Array<MessageTypeUnhydrated>> {
const db = getInstance();
const rows: JSONRows = db
@ -2354,6 +2416,8 @@ async function getNewerMessagesByConversation(
`
SELECT json FROM messages WHERE
conversationId = $conversationId AND
type IS NOT 'story' AND
storyId IS $storyId AND
(
(received_at = $received_at AND sent_at > $sent_at) OR
received_at > $received_at
@ -2364,28 +2428,33 @@ async function getNewerMessagesByConversation(
)
.all({
conversationId,
limit,
received_at: receivedAt,
sent_at: sentAt,
limit,
storyId: storyId || null,
});
return rows;
}
function getOldestMessageForConversation(
conversationId: string
conversationId: string,
storyId?: UUIDStringType
): MessageMetricsType | undefined {
const db = getInstance();
const row = db
.prepare<Query>(
`
SELECT * FROM messages WHERE
conversationId = $conversationId
conversationId = $conversationId AND
type IS NOT 'story' AND
storyId IS $storyId
ORDER BY received_at ASC, sent_at ASC
LIMIT 1;
`
)
.get({
conversationId,
storyId: storyId || null,
});
if (!row) {
@ -2395,20 +2464,24 @@ function getOldestMessageForConversation(
return row;
}
function getNewestMessageForConversation(
conversationId: string
conversationId: string,
storyId?: UUIDStringType
): MessageMetricsType | undefined {
const db = getInstance();
const row = db
.prepare<Query>(
`
SELECT * FROM messages WHERE
conversationId = $conversationId
conversationId = $conversationId AND
type IS NOT 'story' AND
storyId IS $storyId
ORDER BY received_at DESC, sent_at DESC
LIMIT 1;
`
)
.get({
conversationId,
storyId: storyId || null,
});
if (!row) {
@ -2435,13 +2508,14 @@ function getLastConversationActivity({
(type IS NULL
OR
type NOT IN (
'profile-change',
'verified-change',
'message-history-unsynced',
'keychange',
'change-number-notification',
'group-v1-migration',
'keychange',
'message-history-unsynced',
'profile-change',
'story',
'universal-timer-notification',
'change-number-notification'
'verified-change'
)
) AND
(
@ -2492,12 +2566,13 @@ function getLastConversationPreview({
type IS NULL
OR
type NOT IN (
'profile-change',
'verified-change',
'message-history-unsynced',
'change-number-notification',
'group-v1-migration',
'message-history-unsynced',
'profile-change',
'story',
'universal-timer-notification',
'change-number-notification'
'verified-change'
)
) AND NOT
(
@ -2548,7 +2623,8 @@ async function getLastConversationMessages({
}
function getOldestUnreadMessageForConversation(
conversationId: string
conversationId: string,
storyId?: UUIDStringType
): MessageMetricsType | undefined {
const db = getInstance();
const row = db
@ -2556,13 +2632,16 @@ function getOldestUnreadMessageForConversation(
`
SELECT * FROM messages WHERE
conversationId = $conversationId AND
readStatus = ${ReadStatus.Unread}
readStatus = ${ReadStatus.Unread} AND
type IS NOT 'story' AND
storyId IS $storyId
ORDER BY received_at ASC, sent_at ASC
LIMIT 1;
`
)
.get({
conversationId,
storyId: storyId || null,
});
if (!row) {
@ -2572,7 +2651,10 @@ function getOldestUnreadMessageForConversation(
return row;
}
function getTotalUnreadForConversation(conversationId: string): number {
async function getTotalUnreadForConversation(
conversationId: string,
storyId?: UUIDStringType
): Promise<number> {
const db = getInstance();
const row = db
.prepare<Query>(
@ -2581,11 +2663,14 @@ function getTotalUnreadForConversation(conversationId: string): number {
FROM messages
WHERE
conversationId = $conversationId AND
readStatus = ${ReadStatus.Unread};
readStatus = ${ReadStatus.Unread} AND
type IS NOT 'story' AND
storyId IS $storyId;
`
)
.get({
conversationId,
storyId: storyId || null,
});
if (!row) {
@ -2596,12 +2681,19 @@ function getTotalUnreadForConversation(conversationId: string): number {
}
async function getMessageMetricsForConversation(
conversationId: string
conversationId: string,
storyId?: UUIDStringType
): Promise<ConversationMetricsType> {
const oldest = getOldestMessageForConversation(conversationId);
const newest = getNewestMessageForConversation(conversationId);
const oldestUnread = getOldestUnreadMessageForConversation(conversationId);
const totalUnread = getTotalUnreadForConversation(conversationId);
const oldest = getOldestMessageForConversation(conversationId, storyId);
const newest = getNewestMessageForConversation(conversationId, storyId);
const oldestUnread = getOldestUnreadMessageForConversation(
conversationId,
storyId
);
const totalUnread = await getTotalUnreadForConversation(
conversationId,
storyId
);
return {
oldest: oldest ? pick(oldest, ['received_at', 'sent_at', 'id']) : undefined,
@ -3706,29 +3798,256 @@ async function getAllBadgeImageFileLocalPaths(): Promise<Set<string>> {
return new Set(localPaths);
}
type StoryDistributionForDatabase = Readonly<
{
senderKeyInfoJson: string | null;
} & Omit<StoryDistributionType, 'senderKeyInfo'>
>;
function hydrateStoryDistribution(
fromDatabase: StoryDistributionForDatabase
): StoryDistributionType {
return {
...omit(fromDatabase, 'senderKeyInfoJson'),
senderKeyInfo: fromDatabase.senderKeyInfoJson
? JSON.parse(fromDatabase.senderKeyInfoJson)
: undefined,
};
}
function freezeStoryDistribution(
story: StoryDistributionType
): StoryDistributionForDatabase {
return {
...omit(story, 'senderKeyInfo'),
senderKeyInfoJson: story.senderKeyInfo
? JSON.stringify(story.senderKeyInfo)
: null,
};
}
async function _getAllStoryDistributions(): Promise<
Array<StoryDistributionType>
> {
const db = getInstance();
const storyDistributions = db
.prepare<EmptyQuery>('SELECT * FROM storyDistributions;')
.all();
return storyDistributions.map(hydrateStoryDistribution);
}
async function _getAllStoryDistributionMembers(): Promise<
Array<StoryDistributionMemberType>
> {
const db = getInstance();
return db
.prepare<EmptyQuery>('SELECT * FROM storyDistributionMembers;')
.all();
}
async function _deleteAllStoryDistributions(): Promise<void> {
const db = getInstance();
db.prepare<EmptyQuery>('DELETE FROM storyDistributions;').run();
}
async function createNewStoryDistribution(
story: StoryDistributionWithMembersType
): Promise<void> {
const db = getInstance();
db.transaction(() => {
const payload = freezeStoryDistribution(story);
prepare(
db,
`
INSERT INTO storyDistributions(
id,
name,
avatarUrlPath,
avatarKey,
senderKeyInfoJson
) VALUES (
$id,
$name,
$avatarUrlPath,
$avatarKey,
$senderKeyInfoJson
);
`
).run(payload);
const { id: listId, members } = story;
const memberInsertStatement = prepare(
db,
`
INSERT OR REPLACE INTO storyDistributionMembers (
listId,
uuid
) VALUES (
$listId,
$uuid
);
`
);
for (const uuid of members) {
memberInsertStatement.run({
listId,
uuid,
});
}
})();
}
async function getAllStoryDistributionsWithMembers(): Promise<
Array<StoryDistributionWithMembersType>
> {
const allDistributions = await _getAllStoryDistributions();
const allMembers = await _getAllStoryDistributionMembers();
const byListId = groupBy(allMembers, member => member.listId);
return allDistributions.map(list => ({
...list,
members: (byListId[list.id] || []).map(member => member.uuid),
}));
}
async function modifyStoryDistributionMembers(
listId: string,
{
toAdd,
toRemove,
}: { toAdd: Array<UUIDStringType>; toRemove: Array<UUIDStringType> }
): Promise<void> {
const db = getInstance();
db.transaction(() => {
const memberInsertStatement = prepare(
db,
`
INSERT OR REPLACE INTO storyDistributionMembers (
listId,
uuid
) VALUES (
$listId,
$uuid
);
`
);
for (const uuid of toAdd) {
memberInsertStatement.run({
listId,
uuid,
});
}
batchMultiVarQuery(db, toRemove, (uuids: Array<UUIDStringType>) => {
db.prepare<ArrayQuery>(
`
DELETE FROM storyDistributionMembers
WHERE listId = ? AND uuid IN ( ${uuids.map(() => '?').join(', ')} );
`
).run([listId, ...uuids]);
});
})();
}
async function deleteStoryDistribution(id: UUIDStringType): Promise<void> {
const db = getInstance();
db.prepare<Query>('DELETE FROM storyDistributions WHERE id = $id;').run({
id,
});
}
async function _getAllStoryReads(): Promise<Array<StoryReadType>> {
const db = getInstance();
return db.prepare<EmptyQuery>('SELECT * FROM storyReads;').all();
}
async function _deleteAllStoryReads(): Promise<void> {
const db = getInstance();
db.prepare<EmptyQuery>('DELETE FROM storyReads;').run();
}
async function addNewStoryRead(read: StoryReadType): Promise<void> {
const db = getInstance();
prepare(
db,
`
INSERT OR REPLACE INTO storyReads(
authorId,
conversationId,
storyId,
storyReadDate
) VALUES (
$authorId,
$conversationId,
$storyId,
$storyReadDate
);
`
).run(read);
}
async function getLastStoryReadsForAuthor({
authorId,
conversationId,
limit: initialLimit,
}: {
authorId: UUIDStringType;
conversationId?: UUIDStringType;
limit?: number;
}): Promise<Array<StoryReadType>> {
const limit = initialLimit || 5;
const db = getInstance();
return db
.prepare<Query>(
`
SELECT * FROM storyReads
WHERE
authorId = $authorId AND
($conversationId IS NULL OR conversationId = $conversationId)
ORDER BY storyReadDate DESC
LIMIT $limit;
`
)
.all({
authorId,
conversationId: conversationId || null,
limit,
});
}
// All data in database
async function removeAll(): Promise<void> {
const db = getInstance();
db.transaction(() => {
db.exec(`
DELETE FROM badges;
DELETE FROM attachment_downloads;
DELETE FROM badgeImageFiles;
DELETE FROM badges;
DELETE FROM conversations;
DELETE FROM emojis;
DELETE FROM groupCallRings;
DELETE FROM identityKeys;
DELETE FROM items;
DELETE FROM jobs;
DELETE FROM jobs;
DELETE FROM messages_fts;
DELETE FROM messages;
DELETE FROM preKeys;
DELETE FROM reactions;
DELETE FROM senderKeys;
DELETE FROM sendLogMessageIds;
DELETE FROM sendLogPayloads;
DELETE FROM sendLogRecipients;
DELETE FROM sessions;
DELETE FROM signedPreKeys;
DELETE FROM unprocessed;
DELETE FROM attachment_downloads;
DELETE FROM messages_fts;
DELETE FROM stickers;
DELETE FROM sticker_packs;
DELETE FROM sticker_references;
DELETE FROM jobs;
DELETE FROM stickers;
DELETE FROM storyDistributionMembers;
DELETE FROM storyDistributions;
DELETE FROM storyReads;
DELETE FROM unprocessed;
`);
})();
}
@ -3743,12 +4062,15 @@ async function removeAllConfiguration(
db.exec(
`
DELETE FROM identityKeys;
DELETE FROM jobs;
DELETE FROM preKeys;
DELETE FROM senderKeys;
DELETE FROM sendLogMessageIds;
DELETE FROM sendLogPayloads;
DELETE FROM sendLogRecipients;
DELETE FROM sessions;
DELETE FROM signedPreKeys;
DELETE FROM unprocessed;
DELETE FROM jobs;
`
);
@ -3811,6 +4133,8 @@ async function getMessagesWithVisualMediaAttachments(
.prepare<Query>(
`
SELECT json FROM messages WHERE
type IS NOT 'story' AND
storyId IS NULL AND
conversationId = $conversationId AND
hasVisualMediaAttachments = 1
ORDER BY received_at DESC, sent_at DESC
@ -3834,6 +4158,8 @@ async function getMessagesWithFileAttachments(
.prepare<Query>(
`
SELECT json FROM messages WHERE
type IS NOT 'story' AND
storyId IS NULL AND
conversationId = $conversationId AND
hasFileAttachments = 1
ORDER BY received_at DESC, sent_at DESC

133
ts/sql/migrations/45-stories.ts Normal file
View file

@ -0,0 +1,133 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Database } from 'better-sqlite3';
import type { LoggerType } from '../../types/Logging';
export default function updateToSchemaVersion45(
currentVersion: number,
db: Database,
logger: LoggerType
): void {
if (currentVersion >= 45) {
return;
}
db.transaction(() => {
db.exec(
`
--- Add column to messages table
ALTER TABLE messages ADD COLUMN storyId STRING;
--- Update important message indices
DROP INDEX messages_conversation;
CREATE INDEX messages_conversation ON messages
(conversationId, type, storyId, received_at);
DROP INDEX messages_unread;
CREATE INDEX messages_unread ON messages
(conversationId, readStatus, type, storyId) WHERE readStatus IS NOT NULL;
--- Update attachment indices for All Media views
DROP INDEX messages_hasAttachments;
CREATE INDEX messages_hasAttachments
ON messages (conversationId, hasAttachments, received_at)
WHERE type IS NOT 'story' AND storyId IS NULL;
DROP INDEX messages_hasFileAttachments;
CREATE INDEX messages_hasFileAttachments
ON messages (conversationId, hasFileAttachments, received_at)
WHERE type IS NOT 'story' AND storyId IS NULL;
DROP INDEX messages_hasVisualMediaAttachments;
CREATE INDEX messages_hasVisualMediaAttachments
ON messages (conversationId, hasVisualMediaAttachments, received_at)
WHERE type IS NOT 'story' AND storyId IS NULL;
--- Message insert/update triggers to exclude stories and story replies
DROP TRIGGER messages_on_insert;
CREATE TRIGGER messages_on_insert AFTER INSERT ON messages
WHEN new.isViewOnce IS NOT 1 AND new.storyId IS NULL
BEGIN
INSERT INTO messages_fts
(rowid, body)
VALUES
(new.rowid, new.body);
END;
DROP TRIGGER messages_on_update;
CREATE TRIGGER messages_on_update AFTER UPDATE ON messages
WHEN
(new.body IS NULL OR old.body IS NOT new.body) AND
new.isViewOnce IS NOT 1 AND new.storyId IS NULL
BEGIN
DELETE FROM messages_fts WHERE rowid = old.rowid;
INSERT INTO messages_fts
(rowid, body)
VALUES
(new.rowid, new.body);
END;
--- Update delete trigger to remove storyReads
DROP TRIGGER messages_on_delete;
CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
DELETE FROM messages_fts WHERE rowid = old.rowid;
DELETE FROM sendLogPayloads WHERE id IN (
SELECT payloadId FROM sendLogMessageIds
WHERE messageId = old.id
);
DELETE FROM reactions WHERE rowid IN (
SELECT rowid FROM reactions
WHERE messageId = old.id
);
DELETE FROM storyReads WHERE storyId = old.storyId;
END;
--- Story Read History
CREATE TABLE storyReads (
authorId STRING NOT NULL,
conversationId STRING NOT NULL,
storyId STRING NOT NULL,
storyReadDate NUMBER NOT NULL,
PRIMARY KEY (authorId, storyId)
);
CREATE INDEX storyReads_data ON storyReads (
storyReadDate, authorId, conversationId
);
--- Story Distribution Lists
CREATE TABLE storyDistributions(
id STRING PRIMARY KEY NOT NULL,
name TEXT,
avatarUrlPath TEXT,
avatarKey BLOB,
senderKeyInfoJson STRING
);
CREATE TABLE storyDistributionMembers(
listId STRING NOT NULL REFERENCES storyDistributions(id)
ON DELETE CASCADE
ON UPDATE CASCADE,
uuid STRING NOT NULL,
PRIMARY KEY (listId, uuid)
)
`
);
db.pragma('user_version = 45');
})();
logger.info('updateToSchemaVersion45: success!');
}

2
ts/sql/migrations/index.ts
View file

@ -20,6 +20,7 @@ import updateToSchemaVersion41 from './41-uuid-keys';
import updateToSchemaVersion42 from './42-stale-reactions';
import updateToSchemaVersion43 from './43-gv2-uuid';
import updateToSchemaVersion44 from './44-badges';
import updateToSchemaVersion45 from './45-stories';
function updateToSchemaVersion1(
currentVersion: number,
@ -1903,6 +1904,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion42,
updateToSchemaVersion43,
updateToSchemaVersion44,
updateToSchemaVersion45,
];
export function updateSchema(db: Database, logger: LoggerType): void {