mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Also use sent_at whenever we query database with received_at

Browse source
This commit is contained in:
Scott Nonnenberg 2021-01-13 08:32:18 -08:00
parent f32a0b537d
commit 9f81b4157b
8 changed files with 181 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

3
js/modules/backup.js
View file

@ -673,6 +673,7 @@ async function exportConversation(conversation, options = {}) {
// We're looping from the most recent to the oldest // We're looping from the most recent to the oldest
let lastReceivedAt = Number.MAX_VALUE; let lastReceivedAt = Number.MAX_VALUE;
let lastSentAt = Number.MAX_VALUE;
while (!complete) { while (!complete) {
// eslint-disable-next-line no-await-in-loop // eslint-disable-next-line no-await-in-loop
@ -681,6 +682,7 @@ async function exportConversation(conversation, options = {}) {
{ {
limit: CHUNK_SIZE, limit: CHUNK_SIZE,
receivedAt: lastReceivedAt, receivedAt: lastReceivedAt,
sentAt: lastSentAt,
MessageCollection: Whisper.MessageCollection, MessageCollection: Whisper.MessageCollection,
} }
); );
@ -771,6 +773,7 @@ async function exportConversation(conversation, options = {}) {
const last = messages.length > 0 ? messages[messages.length - 1] : null; const last = messages.length > 0 ? messages[messages.length - 1] : null;
if (last) { if (last) {
lastReceivedAt = last.received_at; lastReceivedAt = last.received_at;
lastSentAt = last.sent_at;
} }
if (messages.length < CHUNK_SIZE) { if (messages.length < CHUNK_SIZE) {

1
test/backup_test.js
View file

@ -457,6 +457,7 @@ describe('Backup', () => {
body: 'Totally!', body: 'Totally!',
source: OUR_NUMBER, source: OUR_NUMBER,
received_at: 1524185933350, received_at: 1524185933350,
sent_at: 1524185933350,
timestamp: 1524185933350, timestamp: 1524185933350,
errors: [], errors: [],
attachments: [ attachments: [

1
ts/models/conversations.ts
View file

@ -1314,6 +1314,7 @@ export class ConversationModel extends window.Backbone.Model<
MessageCollection: window.Whisper.MessageCollection, MessageCollection: window.Whisper.MessageCollection,
limit: 100, limit: 100,
receivedAt: first ? first.get('received_at') : undefined, receivedAt: first ? first.get('received_at') : undefined,
sentAt: first ? first.get('sent_at') : undefined,
messageId: first ? first.id : undefined, messageId: first ? first.id : undefined,
} }
); );

6
ts/sql/Client.ts
View file

@ -983,11 +983,13 @@ async function getOlderMessagesByConversation(
{ {
limit = 100, limit = 100,
receivedAt = Number.MAX_VALUE, receivedAt = Number.MAX_VALUE,
sentAt = Number.MAX_VALUE,
messageId, messageId,
MessageCollection, MessageCollection,
}: { }: {
limit?: number; limit?: number;
receivedAt?: number; receivedAt?: number;
sentAt?: number;
messageId?: string; messageId?: string;
MessageCollection: typeof MessageModelCollectionType; MessageCollection: typeof MessageModelCollectionType;
} }
@ -997,6 +999,7 @@ async function getOlderMessagesByConversation(
{ {
limit, limit,
receivedAt, receivedAt,
sentAt,
messageId, messageId,
} }
); );
@ -1008,10 +1011,12 @@ async function getNewerMessagesByConversation(
{ {
limit = 100, limit = 100,
receivedAt = 0, receivedAt = 0,
sentAt = 0,
MessageCollection, MessageCollection,
}: { }: {
limit?: number; limit?: number;
receivedAt?: number; receivedAt?: number;
sentAt?: number;
MessageCollection: typeof MessageModelCollectionType; MessageCollection: typeof MessageModelCollectionType;
} }
) { ) {
@ -1020,6 +1025,7 @@ async function getNewerMessagesByConversation(
{ {
limit, limit,
receivedAt, receivedAt,
sentAt,
} }
); );

5
ts/sql/Interface.ts
View file

@ -217,12 +217,13 @@ export type ServerInterface = DataInterface & {
options?: { options?: {
limit?: number; limit?: number;
receivedAt?: number; receivedAt?: number;
sentAt?: number;
messageId?: string; messageId?: string;
} }
) => Promise<Array<MessageTypeUnhydrated>>; ) => Promise<Array<MessageTypeUnhydrated>>;
getNewerMessagesByConversation: ( getNewerMessagesByConversation: (
conversationId: string, conversationId: string,
options?: { limit?: number; receivedAt?: number } options?: { limit?: number; receivedAt?: number; sentAt?: number }
) => Promise<Array<MessageTypeUnhydrated>>; ) => Promise<Array<MessageTypeUnhydrated>>;
getLastConversationActivity: ( getLastConversationActivity: (
conversationId: string conversationId: string
@ -309,6 +310,7 @@ export type ClientInterface = DataInterface & {
limit?: number; limit?: number;
messageId?: string; messageId?: string;
receivedAt?: number; receivedAt?: number;
sentAt?: number;
MessageCollection: typeof MessageModelCollectionType; MessageCollection: typeof MessageModelCollectionType;
} }
) => Promise<MessageModelCollectionType>; ) => Promise<MessageModelCollectionType>;
@ -317,6 +319,7 @@ export type ClientInterface = DataInterface & {
options: { options: {
limit?: number; limit?: number;
receivedAt?: number; receivedAt?: number;
sentAt?: number;
MessageCollection: typeof MessageModelCollectionType; MessageCollection: typeof MessageModelCollectionType;
} }
) => Promise<MessageModelCollectionType>; ) => Promise<MessageModelCollectionType>;

62
ts/sql/Server.ts
View file

@ -2363,7 +2363,7 @@ async function searchMessages(
INNER JOIN messages on messages_fts.id = messages.id INNER JOIN messages on messages_fts.id = messages.id
WHERE WHERE
messages_fts match $query messages_fts match $query
ORDER BY messages.received_at DESC ORDER BY messages.received_at DESC, messages.sent_at DESC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$query: query, $query: query,
@ -2392,7 +2392,7 @@ async function searchMessagesInConversation(
WHERE WHERE
messages_fts match $query AND messages_fts match $query AND
messages.conversationId = $conversationId messages.conversationId = $conversationId
ORDER BY messages.received_at DESC ORDER BY messages.received_at DESC, messages.sent_at DESC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$query: query, $query: query,
@ -2662,7 +2662,7 @@ async function getUnreadByConversation(conversationId: string) {
`SELECT json FROM messages WHERE `SELECT json FROM messages WHERE
unread = $unread AND unread = $unread AND
conversationId = $conversationId conversationId = $conversationId
ORDER BY received_at DESC;`, ORDER BY received_at DESC, sent_at DESC;`,
{ {
$unread: 1, $unread: 1,
$conversationId: conversationId, $conversationId: conversationId,
@ -2677,10 +2677,12 @@ async function getOlderMessagesByConversation(
{ {
limit = 100, limit = 100,
receivedAt = Number.MAX_VALUE, receivedAt = Number.MAX_VALUE,
sentAt = Number.MAX_VALUE,
messageId, messageId,
}: { }: {
limit?: number; limit?: number;
receivedAt?: number; receivedAt?: number;
sentAt?: number;
messageId?: string; messageId?: string;
} = {} } = {}
) { ) {
@ -2691,13 +2693,17 @@ async function getOlderMessagesByConversation(
rows = await db.all( rows = await db.all(
`SELECT json FROM messages WHERE `SELECT json FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
received_at <= $received_at AND id != $messageId AND
id != $messageId (
ORDER BY received_at DESC (received_at = $received_at AND sent_at < $sent_at) OR
received_at < $received_at
)
ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
$received_at: receivedAt, $received_at: receivedAt,
$sent_at: sentAt,
$limit: limit, $limit: limit,
$messageId: messageId, $messageId: messageId,
} }
@ -2706,12 +2712,16 @@ async function getOlderMessagesByConversation(
rows = await db.all( rows = await db.all(
`SELECT json FROM messages WHERE `SELECT json FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
(
(received_at = $received_at AND sent_at < $sent_at) OR
received_at < $received_at received_at < $received_at
ORDER BY received_at DESC )
ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
$received_at: receivedAt, $received_at: receivedAt,
$sent_at: sentAt,
$limit: limit, $limit: limit,
} }
); );
@ -2722,18 +2732,26 @@ async function getOlderMessagesByConversation(
async function getNewerMessagesByConversation( async function getNewerMessagesByConversation(
conversationId: string, conversationId: string,
{ limit = 100, receivedAt = 0 }: { limit?: number; receivedAt?: number } = {} {
limit = 100,
receivedAt = 0,
sentAt = 0,
}: { limit?: number; receivedAt?: number; sentAt?: number } = {}
) { ) {
const db = getInstance(); const db = getInstance();
const rows = await db.all( const rows = await db.all(
`SELECT json FROM messages WHERE `SELECT json FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
(
(received_at = $received_at AND sent_at > $sent_at) OR
received_at > $received_at received_at > $received_at
ORDER BY received_at ASC )
ORDER BY received_at ASC, sent_at ASC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
$received_at: receivedAt, $received_at: receivedAt,
$sent_at: sentAt,
$limit: limit, $limit: limit,
} }
); );
@ -2745,7 +2763,7 @@ async function getOldestMessageForConversation(conversationId: string) {
const row = await db.get( const row = await db.get(
`SELECT * FROM messages WHERE `SELECT * FROM messages WHERE
conversationId = $conversationId conversationId = $conversationId
ORDER BY received_at ASC ORDER BY received_at ASC, sent_at ASC
LIMIT 1;`, LIMIT 1;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
@ -2763,7 +2781,7 @@ async function getNewestMessageForConversation(conversationId: string) {
const row = await db.get( const row = await db.get(
`SELECT * FROM messages WHERE `SELECT * FROM messages WHERE
conversationId = $conversationId conversationId = $conversationId
ORDER BY received_at DESC ORDER BY received_at DESC, sent_at DESC
LIMIT 1;`, LIMIT 1;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
@ -2786,7 +2804,7 @@ async function getLastConversationActivity(
conversationId = $conversationId AND conversationId = $conversationId AND
(type IS NULL OR type NOT IN ('profile-change', 'verified-change', 'message-history-unsynced', 'keychange', 'group-v1-migration')) AND (type IS NULL OR type NOT IN ('profile-change', 'verified-change', 'message-history-unsynced', 'keychange', 'group-v1-migration')) AND
(json_extract(json, '$.expirationTimerUpdate.fromSync') IS NULL OR json_extract(json, '$.expirationTimerUpdate.fromSync') != 1) (json_extract(json, '$.expirationTimerUpdate.fromSync') IS NULL OR json_extract(json, '$.expirationTimerUpdate.fromSync') != 1)
ORDER BY received_at DESC ORDER BY received_at DESC, sent_at DESC
LIMIT 1;`, LIMIT 1;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
@ -2807,7 +2825,7 @@ async function getLastConversationPreview(
`SELECT * FROM messages WHERE `SELECT * FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
(type IS NULL OR type NOT IN ('profile-change', 'verified-change', 'message-history-unsynced', 'group-v1-migration')) (type IS NULL OR type NOT IN ('profile-change', 'verified-change', 'message-history-unsynced', 'group-v1-migration'))
ORDER BY received_at DESC ORDER BY received_at DESC, sent_at DESC
LIMIT 1;`, LIMIT 1;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
@ -2826,7 +2844,7 @@ async function getOldestUnreadMessageForConversation(conversationId: string) {
`SELECT * FROM messages WHERE `SELECT * FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
unread = 1 unread = 1
ORDER BY received_at ASC ORDER BY received_at ASC, sent_at ASC
LIMIT 1;`, LIMIT 1;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
@ -2870,10 +2888,10 @@ async function getMessageMetricsForConversation(conversationId: string) {
const [oldest, newest, oldestUnread, totalUnread] = results; const [oldest, newest, oldestUnread, totalUnread] = results;
return { return {
oldest: oldest ? pick(oldest, ['received_at', 'id']) : null, oldest: oldest ? pick(oldest, ['received_at', 'sent_at', 'id']) : null,
newest: newest ? pick(newest, ['received_at', 'id']) : null, newest: newest ? pick(newest, ['received_at', 'sent_at', 'id']) : null,
oldestUnread: oldestUnread oldestUnread: oldestUnread
? pick(oldestUnread, ['received_at', 'id']) ? pick(oldestUnread, ['received_at', 'sent_at', 'id'])
: null, : null,
totalUnread, totalUnread,
}; };
@ -2932,7 +2950,7 @@ async function getMessagesBySentAt(sentAt: number) {
const rows = await db.all( const rows = await db.all(
`SELECT * FROM messages `SELECT * FROM messages
WHERE sent_at = $sent_at WHERE sent_at = $sent_at
ORDER BY received_at DESC;`, ORDER BY received_at DESC, sent_at DESC;`,
{ {
$sent_at: sentAt, $sent_at: sentAt,
} }
@ -2998,7 +3016,7 @@ async function getNextTapToViewMessageToAgeOut() {
WHERE WHERE
isViewOnce = 1 isViewOnce = 1
AND (isErased IS NULL OR isErased != 1) AND (isErased IS NULL OR isErased != 1)
ORDER BY received_at ASC ORDER BY received_at ASC, sent_at ASC
LIMIT 1; LIMIT 1;
`); `);
@ -3019,7 +3037,7 @@ async function getTapToViewMessagesNeedingErase() {
isViewOnce = 1 isViewOnce = 1
AND (isErased IS NULL OR isErased != 1) AND (isErased IS NULL OR isErased != 1)
AND received_at <= $THIRTY_DAYS_AGO AND received_at <= $THIRTY_DAYS_AGO
ORDER BY received_at ASC;`, ORDER BY received_at ASC, sent_at ASC;`,
{ {
$THIRTY_DAYS_AGO: THIRTY_DAYS_AGO, $THIRTY_DAYS_AGO: THIRTY_DAYS_AGO,
} }
@ -3824,7 +3842,7 @@ async function getMessagesWithVisualMediaAttachments(
`SELECT json FROM messages WHERE `SELECT json FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
hasVisualMediaAttachments = 1 hasVisualMediaAttachments = 1
ORDER BY received_at DESC ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,
@ -3844,7 +3862,7 @@ async function getMessagesWithFileAttachments(
`SELECT json FROM messages WHERE `SELECT json FROM messages WHERE
conversationId = $conversationId AND conversationId = $conversationId AND
hasFileAttachments = 1 hasFileAttachments = 1
ORDER BY received_at DESC ORDER BY received_at DESC, sent_at DESC
LIMIT $limit;`, LIMIT $limit;`,
{ {
$conversationId: conversationId, $conversationId: conversationId,

28
ts/state/ducks/conversations.ts
View file

@ -160,6 +160,7 @@ export type MessageType = {
type MessagePointerType = { type MessagePointerType = {
id: string; id: string;
received_at: number; received_at: number;
sent_at?: number;
}; };
type MessageMetricsType = { type MessageMetricsType = {
newest?: MessagePointerType; newest?: MessagePointerType;
@ -1011,7 +1012,7 @@ export function reducer(
if (messages.length > 0) { if (messages.length > 0) {
const first = messages[0]; const first = messages[0];
if (first && (!oldest || first.received_at <= oldest.received_at)) { if (first && (!oldest || first.received_at <= oldest.received_at)) {
oldest = pick(first, ['id', 'received_at']); oldest = pick(first, ['id', 'received_at', 'sent_at']);
} }
const last = messages[messages.length - 1]; const last = messages[messages.length - 1];
@ -1019,7 +1020,7 @@ export function reducer(
last && last &&
(!newest || unboundedFetch || last.received_at >= newest.received_at) (!newest || unboundedFetch || last.received_at >= newest.received_at)
) { ) {
newest = pick(last, ['id', 'received_at']); newest = pick(last, ['id', 'received_at', 'sent_at']);
} }
} }
@ -1172,12 +1173,14 @@ export function reducer(
if (oldest && oldest.id === firstId && firstId === id) { if (oldest && oldest.id === firstId && firstId === id) {
const second = messagesLookup[oldIds[1]]; const second = messagesLookup[oldIds[1]];
oldest = second ? pick(second, ['id', 'received_at']) : undefined; oldest = second
? pick(second, ['id', 'received_at', 'sent_at'])
: undefined;
} }
if (newest && newest.id === lastId && lastId === id) { if (newest && newest.id === lastId && lastId === id) {
const penultimate = messagesLookup[oldIds[oldIds.length - 2]]; const penultimate = messagesLookup[oldIds[oldIds.length - 2]];
newest = penultimate newest = penultimate
? pick(penultimate, ['id', 'received_at']) ? pick(penultimate, ['id', 'received_at', 'sent_at'])
: undefined; : undefined;
} }
} }
@ -1231,7 +1234,9 @@ export function reducer(
? messageIds[messageIds.length - 1] ? messageIds[messageIds.length - 1]
: undefined; : undefined;
const last = lastId ? getOwn(messagesLookup, lastId) : undefined; const last = lastId ? getOwn(messagesLookup, lastId) : undefined;
const newest = last ? pick(last, ['id', 'received_at']) : undefined; const newest = last
? pick(last, ['id', 'received_at', 'sent_at'])
: undefined;
return { return {
...state, ...state,
@ -1260,7 +1265,9 @@ export function reducer(
const { messageIds } = existingConversation; const { messageIds } = existingConversation;
const firstId = messageIds && messageIds.length ? messageIds[0] : undefined; const firstId = messageIds && messageIds.length ? messageIds[0] : undefined;
const first = firstId ? getOwn(messagesLookup, firstId) : undefined; const first = firstId ? getOwn(messagesLookup, firstId) : undefined;
const oldest = first ? pick(first, ['id', 'received_at']) : undefined; const oldest = first
? pick(first, ['id', 'received_at', 'sent_at'])
: undefined;
return { return {
...state, ...state,
@ -1315,10 +1322,10 @@ export function reducer(
const last = sorted[sorted.length - 1]; const last = sorted[sorted.length - 1];
if (!newest) { if (!newest) {
newest = pick(first, ['id', 'received_at']); newest = pick(first, ['id', 'received_at', 'sent_at']);
} }
if (!oldest) { if (!oldest) {
oldest = pick(last, ['id', 'received_at']); oldest = pick(last, ['id', 'received_at', 'sent_at']);
} }
const existingTotal = existingConversation.messageIds.length; const existingTotal = existingConversation.messageIds.length;
@ -1336,10 +1343,10 @@ export function reducer(
// Update oldest and newest if we receive older/newer // Update oldest and newest if we receive older/newer
// messages (or duplicated timestamps!) // messages (or duplicated timestamps!)
if (first && oldest && first.received_at <= oldest.received_at) { if (first && oldest && first.received_at <= oldest.received_at) {
oldest = pick(first, ['id', 'received_at']); oldest = pick(first, ['id', 'received_at', 'sent_at']);
} }
if (last && newest && last.received_at >= newest.received_at) { if (last && newest && last.received_at >= newest.received_at) {
newest = pick(last, ['id', 'received_at']); newest = pick(last, ['id', 'received_at', 'sent_at']);
} }
const newIds = messages.map(message => message.id); const newIds = messages.map(message => message.id);
@ -1357,6 +1364,7 @@ export function reducer(
oldestUnread = pick(lookup[oldestId], [ oldestUnread = pick(lookup[oldestId], [
'id', 'id',
'received_at', 'received_at',
'sent_at',
]) as MessagePointerType; ]) as MessagePointerType;
} }
} }

7
ts/views/conversation_view.ts
View file

@ -840,8 +840,10 @@ Whisper.ConversationView = Whisper.View.extend({
} }
const receivedAt = message.get('received_at'); const receivedAt = message.get('received_at');
const sentAt = message.get('sent_at');
const models = await getOlderMessagesByConversation(conversationId, { const models = await getOlderMessagesByConversation(conversationId, {
receivedAt, receivedAt,
sentAt,
messageId: oldestMessageId, messageId: oldestMessageId,
limit: 500, limit: 500,
MessageCollection: Whisper.MessageCollection, MessageCollection: Whisper.MessageCollection,
@ -894,8 +896,10 @@ Whisper.ConversationView = Whisper.View.extend({
} }
const receivedAt = message.get('received_at'); const receivedAt = message.get('received_at');
const sentAt = message.get('sent_at');
const models = await getNewerMessagesByConversation(this.model.id, { const models = await getNewerMessagesByConversation(this.model.id, {
receivedAt, receivedAt,
sentAt,
limit: 500, limit: 500,
MessageCollection: Whisper.MessageCollection, MessageCollection: Whisper.MessageCollection,
}); });
@ -1075,15 +1079,18 @@ Whisper.ConversationView = Whisper.View.extend({
} }
const receivedAt = message.get('received_at'); const receivedAt = message.get('received_at');
const sentAt = message.get('sent_at');
const older = await getOlderMessagesByConversation(conversationId, { const older = await getOlderMessagesByConversation(conversationId, {
limit: 250, limit: 250,
receivedAt, receivedAt,
sentAt,
messageId, messageId,
MessageCollection: Whisper.MessageCollection, MessageCollection: Whisper.MessageCollection,
}); });
const newer = await getNewerMessagesByConversation(conversationId, { const newer = await getNewerMessagesByConversation(conversationId, {
limit: 250, limit: 250,
receivedAt, receivedAt,
sentAt,
MessageCollection: Whisper.MessageCollection, MessageCollection: Whisper.MessageCollection,
}); });
const metrics = await getMessageMetricsForConversation(conversationId); const metrics = await getMessageMetricsForConversation(conversationId);