mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Improve cold start performance

Browse source
This commit is contained in:
Josh Perez 2021-03-04 16:44:57 -05:00 committed by Josh Perez
parent c73e35b1b6
commit d82ce07942
39 changed files with 911 additions and 628 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
about_preload.js
View file

@ -25,4 +25,4 @@ window.closeAbout = () => ipcRenderer.send('close-about');
window.i18n = i18n.setup(locale, localeMessages);
require('./ts/logging/set_up_renderer_logging');
require('./ts/logging/set_up_renderer_logging').initialize();

2
debug_log_preload.js
View file

@ -32,7 +32,7 @@ window.getEnvironment = getEnvironment;
window.Backbone = require('backbone');
require('./ts/backbone/views/whisper_view');
require('./ts/backbone/views/toast_view');
require('./ts/logging/set_up_renderer_logging');
require('./ts/logging/set_up_renderer_logging').initialize();
window.closeDebugLog = () => ipcRenderer.send('close-debug-log');
window.Backbone = require('backbone');

50
js/expiring_messages.js
View file

@ -18,32 +18,38 @@
MessageCollection: Whisper.MessageCollection,
});
await Promise.all(
messages.map(async fromDB => {
const message = MessageController.register(fromDB.id, fromDB);
const messageIds = [];
const inMemoryMessages = [];
const messageCleanup = [];
window.log.info('Message expired', {
sentAt: message.get('sent_at'),
});
messages.forEach(dbMessage => {
const message = MessageController.register(dbMessage.id, dbMessage);
messageIds.push(message.id);
inMemoryMessages.push(message);
messageCleanup.push(message.cleanup());
});
// We delete after the trigger to allow the conversation time to process
// the expiration before the message is removed from the database.
await window.Signal.Data.removeMessage(message.id, {
Message: Whisper.Message,
});
// We delete after the trigger to allow the conversation time to process
// the expiration before the message is removed from the database.
await window.Signal.Data.removeMessages(messageIds);
await Promise.all(messageCleanup);
Whisper.events.trigger(
'messageExpired',
message.id,
message.conversationId
);
inMemoryMessages.forEach(message => {
window.log.info('Message expired', {
sentAt: message.get('sent_at'),
});
const conversation = message.getConversation();
if (conversation) {
conversation.trigger('expired', message);
}
})
);
Whisper.events.trigger(
'messageExpired',
message.id,
message.conversationId
);
const conversation = message.getConversation();
if (conversation) {
conversation.trigger('expired', message);
}
});
} catch (error) {
window.log.error(
'destroyExpiredMessages: Error deleting expired messages',

4
js/expiring_tap_to_view_messages.js
View file

@ -58,7 +58,9 @@
return;
}
const nextCheck = toAgeOut.get('received_at') + THIRTY_DAYS;
const receivedAt =
toAgeOut.get('received_at_ms') || toAgeOut.get('received_at');
const nextCheck = receivedAt + THIRTY_DAYS;
Whisper.TapToViewMessagesListener.nextCheck = nextCheck;
window.log.info(

30
js/message_controller.js
View file

@ -6,6 +6,8 @@
window.Whisper = window.Whisper || {};
const messageLookup = Object.create(null);
const msgIDsBySender = new Map();
const msgIDsBySentAt = new Map();
const SECOND = 1000;
const MINUTE = SECOND * 60;
@ -31,10 +33,18 @@
timestamp: Date.now(),
};
msgIDsBySentAt.set(message.get('sent_at'), id);
msgIDsBySender.set(message.getSenderIdentifier(), id);
return message;
}
function unregister(id) {
const { message } = messageLookup[id] || {};
if (message) {
msgIDsBySender.delete(message.getSenderIdentifier());
msgIDsBySentAt.delete(message.get('sent_at'));
}
delete messageLookup[id];
}
@ -50,7 +60,7 @@
now - timestamp > FIVE_MINUTES &&
(!conversation || !conversation.messageCollection.length)
) {
delete messageLookup[message.id];
unregister(message.id);
}
}
}
@ -60,6 +70,22 @@
return existing && existing.message ? existing.message : null;
}
function findBySentAt(sentAt) {
const id = msgIDsBySentAt.get(sentAt);
if (!id) {
return null;
}
return getById(id);
}
function findBySender(sender) {
const id = msgIDsBySender.get(sender);
if (!id) {
return null;
}
return getById(id);
}
function _get() {
return messageLookup;
}
@ -70,6 +96,8 @@
register,
unregister,
cleanup,
findBySender,
findBySentAt,
getById,
_get,
};

13
libtextsecure/test/_test.js
View file

@ -67,3 +67,16 @@ window.Whisper.events = {
on() {},
trigger() {},
};
before(async () => {
try {
window.log.info('Initializing SQL in renderer');
await window.sqlInitializer.initialize();
window.log.info('SQL initialized in renderer');
} catch (err) {
window.log.error(
'SQL failed to initialize',
err && err.stack ? err.stack : err
);
}
});

53
main.js
View file

@ -19,6 +19,8 @@ const electron = require('electron');
const packageJson = require('./package.json');
const GlobalErrors = require('./app/global_errors');
const { setup: setupSpellChecker } = require('./app/spell_check');
const { redactAll } = require('./js/modules/privacy');
const removeUserConfig = require('./app/user_config').remove;
GlobalErrors.addHandler();
@ -30,6 +32,7 @@ const getRealPath = pify(fs.realpath);
const {
app,
BrowserWindow,
clipboard,
dialog,
ipcMain: ipc,
Menu,
@ -1058,12 +1061,37 @@ app.on('ready', async () => {
loadingWindow.loadURL(prepareURL([__dirname, 'loading.html']));
});
const success = await sqlInitPromise;
if (!success) {
try {
await sqlInitPromise;
} catch (error) {
console.log('sql.initialize was unsuccessful; returning early');
const buttonIndex = dialog.showMessageBoxSync({
buttons: [
locale.messages.copyErrorAndQuit.message,
locale.messages.deleteAndRestart.message,
],
defaultId: 0,
detail: redactAll(error.stack),
message: locale.messages.databaseError.message,
noLink: true,
type: 'error',
});
if (buttonIndex === 0) {
clipboard.writeText(
`Database startup error:\n\n${redactAll(error.stack)}`
);
} else {
await sql.removeDB();
removeUserConfig();
app.relaunch();
}
app.exit(1);
return;
}
// eslint-disable-next-line more/no-then
appStartInitialSpellcheckSetting = await getSpellCheckSetting();
await sqlChannels.initialize();
@ -1075,10 +1103,10 @@ app.on('ready', async () => {
await sql.removeIndexedDBFiles();
await sql.removeItemById(IDB_KEY);
}
} catch (error) {
} catch (err) {
console.log(
'(ready event handler) error deleting IndexedDB:',
error && error.stack ? error.stack : error
err && err.stack ? err.stack : err
);
}
@ -1113,10 +1141,10 @@ app.on('ready', async () => {
try {
await attachments.clearTempPath(userDataPath);
} catch (error) {
} catch (err) {
logger.error(
'main/ready: Error deleting temp dir:',
error && error.stack ? error.stack : error
err && err.stack ? err.stack : err
);
}
await attachmentChannel.initialize({
@ -1458,6 +1486,17 @@ ipc.on('locale-data', event => {
event.returnValue = locale.messages;
});
// Used once to initialize SQL in the renderer process
ipc.once('user-config-key', event => {
// eslint-disable-next-line no-param-reassign
event.returnValue = userConfig.get('key');
});
ipc.on('get-user-data-path', event => {
// eslint-disable-next-line no-param-reassign
event.returnValue = app.getPath('userData');
});
function getDataFromMainWindow(name, callback) {
ipc.once(`get-success-${name}`, (_event, error, value) =>
callback(error, value)

2
permissions_popup_preload.js
View file

@ -49,7 +49,7 @@ window.subscribeToSystemThemeChange = fn => {
});
};
require('./ts/logging/set_up_renderer_logging');
require('./ts/logging/set_up_renderer_logging').initialize();
window.closePermissionsPopup = () =>
ipcRenderer.send('close-permissions-popup');

4
preload.js
View file

@ -22,6 +22,8 @@ try {
const { app } = remote;
const { nativeTheme } = remote.require('electron');
window.sqlInitializer = require('./ts/sql/initialize');
window.PROTO_ROOT = 'protos';
const config = require('url').parse(window.location.toString(), true).query;
@ -400,7 +402,7 @@ try {
// We pull these dependencies in now, from here, because they have Node.js dependencies
require('./ts/logging/set_up_renderer_logging');
require('./ts/logging/set_up_renderer_logging').initialize();
if (config.proxyUrl) {
window.log.info('Using provided proxy url');

2
settings_preload.js
View file

@ -129,4 +129,4 @@ function makeSetter(name) {
window.Backbone = require('backbone');
require('./ts/backbone/views/whisper_view');
require('./ts/backbone/views/toast_view');
require('./ts/logging/set_up_renderer_logging');
require('./ts/logging/set_up_renderer_logging').initialize();

10
test/_test.js
View file

@ -76,6 +76,16 @@ function deleteIndexedDB() {
/* Delete the database before running any tests */
before(async () => {
await deleteIndexedDB();
try {
window.log.info('Initializing SQL in renderer');
await window.sqlInitializer.initialize();
window.log.info('SQL initialized in renderer');
} catch (err) {
window.log.error(
'SQL failed to initialize',
err && err.stack ? err.stack : err
);
}
await window.Signal.Data.removeAll();
await window.storage.fetch();
});

48
ts/background.ts
View file

@ -3,8 +3,19 @@
import { DataMessageClass } from './textsecure.d';
import { WhatIsThis } from './window.d';
import { assert } from './util/assert';
export async function startApp(): Promise<void> {
try {
window.log.info('Initializing SQL in renderer');
await window.sqlInitializer.initialize();
window.log.info('SQL initialized in renderer');
} catch (err) {
window.log.error(
'SQL failed to initialize',
err && err.stack ? err.stack : err
);
}
const eventHandlerQueue = new window.PQueue({
concurrency: 1,
timeout: 1000 * 60 * 2,
@ -1615,6 +1626,9 @@ export async function startApp(): Promise<void> {
let connectCount = 0;
let connecting = false;
async function connect(firstRun?: boolean) {
window.receivedAtCounter =
window.storage.get('lastReceivedAtCounter') || Date.now();
if (connecting) {
window.log.warn('connect already running', { connectCount });
return;
@ -2038,7 +2052,7 @@ export async function startApp(): Promise<void> {
logger: window.log,
});
let interval: NodeJS.Timer | null = setInterval(() => {
let interval: NodeJS.Timer | null = setInterval(async () => {
const view = window.owsDesktopApp.appView;
if (view) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@ -2046,6 +2060,24 @@ export async function startApp(): Promise<void> {
interval = null;
view.onEmpty();
window.logAppLoadedEvent();
const attachmentDownloadQueue = window.attachmentDownloadQueue || [];
const THREE_DAYS_AGO = Date.now() - 3600 * 72 * 1000;
const MAX_ATTACHMENT_MSGS_TO_DOWNLOAD = 250;
const attachmentsToDownload = attachmentDownloadQueue.filter(
(message, index) =>
index <= MAX_ATTACHMENT_MSGS_TO_DOWNLOAD ||
message.getReceivedAt() < THREE_DAYS_AGO
);
window.log.info(
'Downloading recent attachments of total attachments',
attachmentsToDownload.length,
attachmentDownloadQueue.length
);
await Promise.all(
attachmentsToDownload.map(message =>
message.queueAttachmentDownloads()
)
);
}
}, 500);
@ -2646,14 +2678,15 @@ export async function startApp(): Promise<void> {
sent_at: data.timestamp,
serverTimestamp: data.serverTimestamp,
sent_to: sentTo,
received_at: now,
received_at: data.receivedAtCounter,
received_at_ms: data.receivedAtDate,
conversationId: descriptor.id,
type: 'outgoing',
sent: true,
unidentifiedDeliveries: data.unidentifiedDeliveries || [],
expirationStartTimestamp: Math.min(
data.expirationStartTimestamp || data.timestamp || Date.now(),
Date.now()
data.expirationStartTimestamp || data.timestamp || now,
now
),
} as WhatIsThis);
}
@ -2856,13 +2889,18 @@ export async function startApp(): Promise<void> {
data: WhatIsThis,
descriptor: MessageDescriptor
) {
assert(
Boolean(data.receivedAtCounter),
`Did not receive receivedAtCounter for message: ${data.timestamp}`
);
return new window.Whisper.Message({
source: data.source,
sourceUuid: data.sourceUuid,
sourceDevice: data.sourceDevice,
sent_at: data.timestamp,
serverTimestamp: data.serverTimestamp,
received_at: Date.now(),
received_at: data.receivedAtCounter,
received_at_ms: data.receivedAtDate,
conversationId: descriptor.id,
unidentifiedDeliveryReceived: data.unidentifiedDeliveryReceived,
type: 'incoming',

9
ts/components/LightboxGallery.stories.tsx
View file

@ -40,7 +40,8 @@ story.add('Image and Video', () => {
message: {
attachments: [],
id: 'image-msg',
received_at: Date.now(),
received_at: 1,
received_at_ms: Date.now(),
},
objectURL: '/fixtures/tina-rolf-269345-unsplash.jpg',
},
@ -55,7 +56,8 @@ story.add('Image and Video', () => {
message: {
attachments: [],
id: 'video-msg',
received_at: Date.now(),
received_at: 2,
received_at_ms: Date.now(),
},
objectURL: '/fixtures/pixabay-Soap-Bubble-7141.mp4',
},
@ -79,7 +81,8 @@ story.add('Missing Media', () => {
message: {
attachments: [],
id: 'image-msg',
received_at: Date.now(),
received_at: 3,
received_at_ms: Date.now(),
},
objectURL: undefined,
},

3
ts/components/conversation/media-gallery/AttachmentSection.stories.tsx
View file

@ -52,7 +52,8 @@ const createRandomFile = (
contentType,
message: {
id: random(now).toString(),
received_at: random(startTime, startTime + timeWindow),
received_at: Math.floor(Math.random() * 10),
received_at_ms: random(startTime, startTime + timeWindow),
attachments: [],
},
attachment: {

3
ts/components/conversation/media-gallery/AttachmentSection.tsx
View file

@ -9,6 +9,7 @@ import { MediaGridItem } from './MediaGridItem';
import { MediaItemType } from '../../LightboxGallery';
import { missingCaseError } from '../../../util/missingCaseError';
import { LocalizerType } from '../../../types/Util';
import { getMessageTimestamp } from '../../../util/getMessageTimestamp';
export type Props = {
i18n: LocalizerType;
@ -58,7 +59,7 @@ export class AttachmentSection extends React.Component<Props> {
fileSize={attachment.size}
shouldShowSeparator={shouldShowSeparator}
onClick={onClick}
timestamp={message.received_at}
timestamp={getMessageTimestamp(message)}
/>
);
default:

3
ts/components/conversation/media-gallery/MediaGallery.tsx
View file

@ -12,6 +12,7 @@ import { groupMediaItemsByDate } from './groupMediaItemsByDate';
import { ItemClickEvent } from './types/ItemClickEvent';
import { missingCaseError } from '../../../util/missingCaseError';
import { LocalizerType } from '../../../types/Util';
import { getMessageTimestamp } from '../../../util/getMessageTimestamp';
import { MediaItemType } from '../../LightboxGallery';
@ -145,7 +146,7 @@ export class MediaGallery extends React.Component<Props, State> {
const sections = groupMediaItemsByDate(now, mediaItems).map(section => {
const first = section.mediaItems[0];
const { message } = first;
const date = moment(message.received_at);
const date = moment(getMessageTimestamp(message));
const header =
section.type === 'yearMonth'
? date.format(MONTH_FORMAT)

3
ts/components/conversation/media-gallery/groupMediaItemsByDate.ts
View file

@ -5,6 +5,7 @@ import moment from 'moment';
import { compact, groupBy, sortBy } from 'lodash';
import { MediaItemType } from '../../LightboxGallery';
import { getMessageTimestamp } from '../../../util/getMessageTimestamp';
// import { missingCaseError } from '../../../util/missingCaseError';
@ -120,7 +121,7 @@ const withSection = (referenceDateTime: moment.Moment) => (
const thisMonth = moment(referenceDateTime).startOf('month');
const { message } = mediaItem;
const mediaItemReceivedDate = moment.utc(message.received_at);
const mediaItemReceivedDate = moment.utc(getMessageTimestamp(message));
if (mediaItemReceivedDate.isAfter(today)) {
return {
order: 0,

2
ts/components/conversation/media-gallery/types/Message.ts
View file

@ -9,4 +9,6 @@ export type Message = {
// Assuming this is for the API
// eslint-disable-next-line camelcase
received_at: number;
// eslint-disable-next-line camelcase
received_at_ms: number;
};

3
ts/groups.ts
View file

@ -2452,7 +2452,8 @@ async function updateGroup({
return {
...changeMessage,
conversationId: conversation.id,
received_at: finalReceivedAt,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: finalReceivedAt,
sent_at: syntheticSentAt,
};
});

29
ts/logging/main_process_logging.ts
View file

@ -61,9 +61,9 @@ export async function initialize(): Promise<bunyan> {
}, 500);
}
const logFile = path.join(logPath, 'log.log');
const logFile = path.join(logPath, 'main.log');
const loggerOptions: bunyan.LoggerOptions = {
name: 'log',
name: 'main',
streams: [
{
type: 'rotating-file',
@ -83,31 +83,6 @@ export async function initialize(): Promise<bunyan> {
const logger = bunyan.createLogger(loggerOptions);
ipc.on('batch-log', (_first, batch: unknown) => {
if (!Array.isArray(batch)) {
logger.error(
'batch-log IPC event was called with a non-array; dropping logs'
);
return;
}
batch.forEach(item => {
if (isLogEntry(item)) {
const levelString = getLogLevelString(item.level);
logger[levelString](
{
time: item.time,
},
item.msg
);
} else {
logger.error(
'batch-log IPC event was called with an invalid log entry; dropping entry'
);
}
});
});
ipc.on('fetch-log', event => {
fetch(logPath).then(
data => {

53
ts/logging/set_up_renderer_logging.ts
View file

@ -7,11 +7,11 @@
import { ipcRenderer as ipc } from 'electron';
import _ from 'lodash';
import { levelFromName } from 'bunyan';
import * as path from 'path';
import * as bunyan from 'bunyan';
import { uploadDebugLogs } from './debuglogs';
import { redactAll } from '../../js/modules/privacy';
import { createBatcher } from '../util/batcher';
import {
LogEntryType,
LogLevel,
@ -23,7 +23,7 @@ import * as log from './log';
import { reallyJsonStringify } from '../util/reallyJsonStringify';
// To make it easier to visually scan logs, we make all levels the same length
const levelMaxLength: number = Object.keys(levelFromName).reduce(
const levelMaxLength: number = Object.keys(bunyan.levelFromName).reduce(
(maxLength, level) => Math.max(maxLength, level.length),
0
);
@ -96,6 +96,30 @@ function fetch(): Promise<string> {
});
}
let globalLogger: undefined | bunyan;
export function initialize(): void {
if (globalLogger) {
throw new Error('Already called initialize!');
}
const basePath = ipc.sendSync('get-user-data-path');
const logFile = path.join(basePath, 'logs', 'app.log');
const loggerOptions: bunyan.LoggerOptions = {
name: 'app',
streams: [
{
type: 'rotating-file',
path: logFile,
period: '1d',
count: 3,
},
],
};
globalLogger = bunyan.createLogger(loggerOptions);
}
const publish = uploadDebugLogs;
// A modern logging interface for the browser
@ -103,14 +127,6 @@ const publish = uploadDebugLogs;
const env = window.getEnvironment();
const IS_PRODUCTION = env === 'production';
const ipcBatcher = createBatcher({
wait: 500,
maxSize: 500,
processBatch: (items: Array<LogEntryType>) => {
ipc.send('batch-log', items);
},
});
// The Bunyan API: https://github.com/trentm/node-bunyan#log-method-api
function logAtLevel(level: LogLevel, ...args: ReadonlyArray<unknown>): void {
if (!IS_PRODUCTION) {
@ -120,11 +136,16 @@ function logAtLevel(level: LogLevel, ...args: ReadonlyArray<unknown>): void {
console._log(prefix, now(), ...args);
}
ipcBatcher.add({
level,
msg: cleanArgs(args),
time: new Date().toISOString(),
});
const levelString = getLogLevelString(level);
const msg = cleanArgs(args);
const time = new Date().toISOString();
if (!globalLogger) {
throw new Error('Logger has not been initialized yet');
return;
}
globalLogger[levelString]({ time }, msg);
}
log.setLogAtLevel(logAtLevel);

1
ts/model-types.d.ts vendored
View file

@ -134,6 +134,7 @@ export type MessageAttributesType = {
groupV2Change?: GroupV2ChangeType;
// Required. Used to sort messages in the database for the conversation timeline.
received_at?: number;
received_at_ms?: number;
// More of a legacy feature, needed as we were updating the schema of messages in the
// background, when we were still in IndexedDB, before attachments had gone to disk
// We set this so that the idle message upgrade process doesn't pick this message up

39
ts/models/conversations.ts
View file

@ -2282,7 +2282,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
type: 'chat-session-refreshed',
sent_at: receivedAt,
received_at: receivedAt,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: receivedAt,
unread: 1,
// TODO: DESKTOP-722
// this type does not fully implement the interface it is expected to
@ -2315,7 +2316,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
type: 'keychange',
sent_at: this.get('timestamp'),
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
key_changed: keyChangedId,
unread: 1,
// TODO: DESKTOP-722
@ -2373,7 +2375,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
type: 'verified-change',
sent_at: lastMessage,
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
verifiedChanged: verifiedChangeId,
verified,
local: options.local,
@ -2435,7 +2438,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
type: 'call-history',
sent_at: timestamp,
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
unread,
callHistoryDetails: detailsToSave,
// TODO: DESKTOP-722
@ -2481,11 +2485,13 @@ export class ConversationModel extends window.Backbone.Model<
profileChange: unknown,
conversationId?: string
): Promise<void> {
const now = Date.now();
const message = ({
conversationId: this.id,
type: 'profile-change',
sent_at: Date.now(),
received_at: Date.now(),
sent_at: now,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: now,
unread: true,
changedId: conversationId || this.id,
profileChange,
@ -2984,7 +2990,8 @@ export class ConversationModel extends window.Backbone.Model<
type: 'outgoing',
conversationId: this.get('id'),
sent_at: timestamp,
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
recipients,
deletedForEveryoneTimestamp: targetTimestamp,
// TODO: DESKTOP-722
@ -3093,7 +3100,8 @@ export class ConversationModel extends window.Backbone.Model<
type: 'outgoing',
conversationId: this.get('id'),
sent_at: timestamp,
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
recipients,
reaction: outgoingReaction,
// TODO: DESKTOP-722
@ -3244,7 +3252,8 @@ export class ConversationModel extends window.Backbone.Model<
preview,
attachments,
sent_at: now,
received_at: now,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: now,
expireTimer,
recipients,
sticker,
@ -3866,7 +3875,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
// No type; 'incoming' messages are specially treated by conversation.markRead()
sent_at: timestamp,
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
flags:
window.textsecure.protobuf.DataMessage.Flags.EXPIRATION_TIMER_UPDATE,
expirationTimerUpdate: {
@ -3970,7 +3980,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
// No type; 'incoming' messages are specially treated by conversation.markRead()
sent_at: timestamp,
received_at: timestamp,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: timestamp,
// TODO: DESKTOP-722
} as unknown) as MessageAttributesType);
@ -4003,7 +4014,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
type: 'outgoing',
sent_at: now,
received_at: now,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: now,
destination: this.get('e164'),
destinationUuid: this.get('uuid'),
recipients: this.getRecipients(),
@ -4059,7 +4071,8 @@ export class ConversationModel extends window.Backbone.Model<
conversationId: this.id,
type: 'outgoing',
sent_at: now,
received_at: now,
received_at: window.Signal.Util.incrementMessageCounter(),
received_at_ms: now,
// TODO: DESKTOP-722
} as unknown) as MessageAttributesType);

138
ts/models/messages.ts
View file

@ -198,6 +198,29 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
};
}
getSenderIdentifier(): string {
const sentAt = this.get('sent_at');
const source = this.get('source');
const sourceUuid = this.get('sourceUuid');
const sourceDevice = this.get('sourceDevice');
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const sourceId = window.ConversationController.ensureContactIds({
e164: source,
uuid: sourceUuid,
})!;
return `${sourceId}.${sourceDevice}-${sentAt}`;
}
getReceivedAt(): number {
// We would like to get the received_at_ms ideally since received_at is
// now an incrementing counter for messages and not the actual time that
// the message was received. If this field doesn't exist on the message
// then we can trust received_at.
return Number(this.get('received_at_ms') || this.get('received_at'));
}
isNormalBubble(): boolean {
return (
!this.isCallHistory() &&
@ -381,7 +404,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
return {
sentAt: this.get('sent_at'),
receivedAt: this.get('received_at'),
receivedAt: this.getReceivedAt(),
message: {
...this.getPropsForMessage(),
disableMenu: true,
@ -1904,9 +1927,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
window.Whisper.Notifications.removeBy({ messageId: this.id });
if (!skipSave) {
await window.Signal.Data.saveMessage(this.attributes, {
Message: window.Whisper.Message,
});
window.Signal.Util.updateMessageBatcher.add(this.attributes);
}
}
@ -1955,9 +1976,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
const id = this.get('id');
if (id && !skipSave) {
await window.Signal.Data.saveMessage(this.attributes, {
Message: window.Whisper.Message,
});
window.Signal.Util.updateMessageBatcher.add(this.attributes);
}
}
}
@ -2822,20 +2841,32 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
}
);
const collection = await window.Signal.Data.getMessagesBySentAt(id, {
MessageCollection: window.Whisper.MessageCollection,
});
const found = collection.find(item => {
const messageAuthorId = item.getContactId();
const inMemoryMessage = window.MessageController.findBySentAt(id);
return authorConversationId === messageAuthorId;
});
let queryMessage;
if (!found) {
quote.referencedMessageNotFound = true;
return message;
if (inMemoryMessage) {
queryMessage = inMemoryMessage;
} else {
window.log.info('copyFromQuotedMessage: db lookup needed', id);
const collection = await window.Signal.Data.getMessagesBySentAt(id, {
MessageCollection: window.Whisper.MessageCollection,
});
const found = collection.find(item => {
const messageAuthorId = item.getContactId();
return authorConversationId === messageAuthorId;
});
if (!found) {
quote.referencedMessageNotFound = true;
return message;
}
queryMessage = window.MessageController.register(found.id, found);
}
if (found.isTapToView()) {
if (queryMessage.isTapToView()) {
quote.text = null;
quote.attachments = [
{
@ -2846,7 +2877,6 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
return message;
}
const queryMessage = window.MessageController.register(found.id, found);
quote.text = queryMessage.get('body');
if (firstAttachment) {
firstAttachment.thumbnail = null;
@ -2946,9 +2976,20 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
);
// First, check for duplicates. If we find one, stop processing here.
const existingMessage = await getMessageBySender(this.attributes, {
Message: window.Whisper.Message,
});
const inMemoryMessage = window.MessageController.findBySender(
this.getSenderIdentifier()
);
if (!inMemoryMessage) {
window.log.info(
'handleDataMessage: duplicate check db lookup needed',
this.getSenderIdentifier()
);
}
const existingMessage =
inMemoryMessage ||
(await getMessageBySender(this.attributes, {
Message: window.Whisper.Message,
}));
const isUpdate = Boolean(data && data.isRecipientUpdate);
if (existingMessage && type === 'incoming') {
@ -3422,7 +3463,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
dataMessage.expireTimer,
source,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
message.get('received_at')!,
message.getReceivedAt()!,
{
fromGroupUpdate: message.isGroupUpdate(),
}
@ -3437,7 +3478,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
undefined,
source,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
message.get('received_at')!
message.getReceivedAt()!
);
}
}
@ -3573,7 +3614,8 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
(this.getConversation()!.getAccepted() || message.isOutgoing()) &&
!shouldHoldOffDownload
) {
await message.queueAttachmentDownloads();
window.attachmentDownloadQueue = window.attachmentDownloadQueue || [];
window.attachmentDownloadQueue.unshift(message);
}
// Does this message have any pending, previously-received associated reactions?
@ -3591,24 +3633,11 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
)
);
await window.Signal.Data.saveMessage(message.attributes, {
Message: window.Whisper.Message,
forceSave: true,
});
conversation.trigger('newmessage', message);
if (message.get('unread')) {
await conversation.notify(message);
}
// Increment the sent message count if this is an outgoing message
if (type === 'outgoing') {
conversation.incrementSentMessageCount();
}
window.Whisper.events.trigger('incrementProgress');
confirm();
window.log.info(
'handleDataMessage: Batching save for',
message.get('sent_at')
);
this.saveAndNotify(conversation, confirm);
} catch (error) {
const errorForLog = error && error.stack ? error.stack : error;
window.log.error(
@ -3622,6 +3651,29 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
});
}
async saveAndNotify(
conversation: ConversationModel,
confirm: () => void
): Promise<void> {
await window.Signal.Util.saveNewMessageBatcher.add(this.attributes);
window.log.info('Message saved', this.get('sent_at'));
conversation.trigger('newmessage', this);
if (this.get('unread')) {
await conversation.notify(this);
}
// Increment the sent message count if this is an outgoing message
if (this.get('type') === 'outgoing') {
conversation.incrementSentMessageCount();
}
window.Whisper.events.trigger('incrementProgress');
confirm();
}
async handleReaction(
reaction: typeof window.WhatIsThis,
shouldPersist = true

477
ts/sql/Client.ts
View file

File diff suppressed because it is too large Load diff

8
ts/sql/Interface.ts
View file

@ -259,7 +259,12 @@ export type ServerInterface = DataInterface & {
configDir: string;
key: string;
messages: LocaleMessagesType;
}) => Promise<boolean>;
}) => Promise<void>;
initializeRenderer: (options: {
configDir: string;
key: string;
}) => Promise<void>;
removeKnownAttachments: (
allAttachments: Array<string>
@ -393,7 +398,6 @@ export type ClientInterface = DataInterface & {
// Client-side only, and test-only
_removeConversations: (ids: Array<string>) => Promise<void>;
_jobs: { [id: string]: ClientJobType };
};
export type ClientJobType = {

141
ts/sql/Queueing.ts Normal file
View file

@ -0,0 +1,141 @@
// Copyright 2018-2020 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import Queue from 'p-queue';
import { ServerInterface } from './Interface';
let allQueriesDone: () => void | undefined;
let sqlQueries = 0;
let singleQueue: Queue | null = null;
let multipleQueue: Queue | null = null;
// Note: we don't want queue timeouts, because delays here are due to in-progress sql
// operations. For example we might try to start a transaction when the prevous isn't
// done, causing that database operation to fail.
function makeNewSingleQueue(): Queue {
singleQueue = new Queue({ concurrency: 1 });
return singleQueue;
}
function makeNewMultipleQueue(): Queue {
multipleQueue = new Queue({ concurrency: 10 });
return multipleQueue;
}
const DEBUG = false;
function makeSQLJob(
fn: ServerInterface[keyof ServerInterface],
args: Array<unknown>,
callName: keyof ServerInterface
) {
if (DEBUG) {
// eslint-disable-next-line no-console
console.log(`SQL(${callName}) queued`);
}
return async () => {
sqlQueries += 1;
const start = Date.now();
if (DEBUG) {
// eslint-disable-next-line no-console
console.log(`SQL(${callName}) started`);
}
let result;
try {
// Ignoring this error TS2556: Expected 3 arguments, but got 0 or more.
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
result = await fn(...args);
} finally {
sqlQueries -= 1;
if (allQueriesDone && sqlQueries <= 0) {
allQueriesDone();
}
}
const end = Date.now();
const delta = end - start;
if (DEBUG || delta > 10) {
// eslint-disable-next-line no-console
console.log(`SQL(${callName}) succeeded in ${end - start}ms`);
}
return result;
};
}
async function handleCall(
fn: ServerInterface[keyof ServerInterface],
args: Array<unknown>,
callName: keyof ServerInterface
) {
if (!fn) {
throw new Error(`sql channel: ${callName} is not an available function`);
}
let result;
// We queue here to keep multi-query operations atomic. Without it, any multistage
// data operation (even within a BEGIN/COMMIT) can become interleaved, since all
// requests share one database connection.
// A needsSerial method must be run in our single concurrency queue.
if (fn.needsSerial) {
if (singleQueue) {
result = await singleQueue.add(makeSQLJob(fn, args, callName));
} else if (multipleQueue) {
const queue = makeNewSingleQueue();
const multipleQueueLocal = multipleQueue;
queue.add(() => multipleQueueLocal.onIdle());
multipleQueue = null;
result = await queue.add(makeSQLJob(fn, args, callName));
} else {
const queue = makeNewSingleQueue();
result = await queue.add(makeSQLJob(fn, args, callName));
}
} else {
// The request can be parallelized. To keep the same structure as the above block
// we force this section into the 'lonely if' pattern.
// eslint-disable-next-line no-lonely-if
if (multipleQueue) {
result = await multipleQueue.add(makeSQLJob(fn, args, callName));
} else if (singleQueue) {
const queue = makeNewMultipleQueue();
queue.pause();
const singleQueueRef = singleQueue;
singleQueue = null;
const promise = queue.add(makeSQLJob(fn, args, callName));
if (singleQueueRef) {
await singleQueueRef.onIdle();
}
queue.start();
result = await promise;
} else {
const queue = makeNewMultipleQueue();
result = await queue.add(makeSQLJob(fn, args, callName));
}
}
return result;
}
export async function waitForPendingQueries(): Promise<void> {
return new Promise<void>(resolve => {
if (sqlQueries === 0) {
resolve();
} else {
allQueriesDone = () => resolve();
}
});
}
export function applyQueueing(dataInterface: ServerInterface): ServerInterface {
return Object.keys(dataInterface).reduce((acc, callName) => {
const serverInterfaceKey = callName as keyof ServerInterface;
acc[serverInterfaceKey] = async (...args: Array<unknown>) =>
handleCall(dataInterface[serverInterfaceKey], args, serverInterfaceKey);
return acc;
}, {} as ServerInterface);
}

106
ts/sql/Server.ts
View file

@ -14,7 +14,6 @@ import mkdirp from 'mkdirp';
import rimraf from 'rimraf';
import PQueue from 'p-queue';
import sql from '@journeyapps/sqlcipher';
import { app, clipboard, dialog } from 'electron';
import pify from 'pify';
import { v4 as generateUUID } from 'uuid';
@ -31,8 +30,6 @@ import {
pick,
} from 'lodash';
import { redactAll } from '../../js/modules/privacy';
import { remove as removeUserConfig } from '../../app/user_config';
import { combineNames } from '../util/combineNames';
import { GroupV2MemberType } from '../model-types.d';
@ -54,6 +51,7 @@ import {
StickerType,
UnprocessedType,
} from './Interface';
import { applyQueueing } from './Queueing';
declare global {
// We want to extend `Function`'s properties, so we need to use an interface.
@ -195,13 +193,14 @@ const dataInterface: ServerInterface = {
// Server-only
initialize,
initializeRenderer,
removeKnownAttachments,
removeKnownStickers,
removeKnownDraftAttachments,
};
export default dataInterface;
export default applyQueueing(dataInterface);
function objectToJSON(data: any) {
return JSON.stringify(data);
@ -210,6 +209,14 @@ function jsonToObject(json: string): any {
return JSON.parse(json);
}
function isRenderer() {
if (typeof process === 'undefined' || !process) {
return true;
}
return process.type === 'renderer';
}
async function openDatabase(filePath: string): Promise<sql.Database> {
return new Promise((resolve, reject) => {
let instance: sql.Database | undefined;
@ -1702,6 +1709,7 @@ async function updateSchema(instance: PromisifiedSQLDatabase) {
}
let globalInstance: PromisifiedSQLDatabase | undefined;
let globalInstanceRenderer: PromisifiedSQLDatabase | undefined;
let databaseFilePath: string | undefined;
let indexedDBPath: string | undefined;
@ -1788,37 +1796,57 @@ async function initialize({
await getMessageCount();
} catch (error) {
console.log('Database startup error:', error.stack);
const buttonIndex = dialog.showMessageBoxSync({
buttons: [
messages.copyErrorAndQuit.message,
messages.deleteAndRestart.message,
],
defaultId: 0,
detail: redactAll(error.stack),
message: messages.databaseError.message,
noLink: true,
type: 'error',
});
if (buttonIndex === 0) {
clipboard.writeText(
`Database startup error:\n\n${redactAll(error.stack)}`
);
} else {
if (promisified) {
await promisified.close();
}
await removeDB();
removeUserConfig();
app.relaunch();
if (promisified) {
await promisified.close();
}
throw error;
}
}
app.exit(1);
return false;
async function initializeRenderer({
configDir,
key,
}: {
configDir: string;
key: string;
}) {
if (!isRenderer()) {
throw new Error('Cannot call from main process.');
}
if (globalInstanceRenderer) {
throw new Error('Cannot initialize more than once!');
}
if (!isString(configDir)) {
throw new Error('initialize: configDir is required!');
}
if (!isString(key)) {
throw new Error('initialize: key is required!');
}
return true;
if (!indexedDBPath) {
indexedDBPath = join(configDir, 'IndexedDB');
}
const dbDir = join(configDir, 'sql');
if (!databaseFilePath) {
databaseFilePath = join(dbDir, 'db.sqlite');
}
let promisified: PromisifiedSQLDatabase | undefined;
try {
promisified = await openAndSetUpSQLCipher(databaseFilePath, { key });
// At this point we can allow general access to the database
globalInstanceRenderer = promisified;
// test database
await getMessageCount();
} catch (error) {
window.log.error('Database startup error:', error.stack);
throw error;
}
}
async function close() {
@ -1857,6 +1885,13 @@ async function removeIndexedDBFiles() {
}
function getInstance(): PromisifiedSQLDatabase {
if (isRenderer()) {
if (!globalInstanceRenderer) {
throw new Error('getInstance: globalInstanceRenderer not set!');
}
return globalInstanceRenderer;
}
if (!globalInstance) {
throw new Error('getInstance: globalInstance not set!');
}
@ -2285,6 +2320,7 @@ async function updateConversation(data: ConversationType) {
}
);
}
async function updateConversations(array: Array<ConversationType>) {
const db = getInstance();
await db.run('BEGIN TRANSACTION;');
@ -2544,7 +2580,7 @@ async function saveMessage(
`UPDATE messages SET
id = $id,
json = $json,
body = $body,
conversationId = $conversationId,
expirationStartTimestamp = $expirationStartTimestamp,
@ -2616,7 +2652,7 @@ async function saveMessage(
`INSERT INTO messages (
id,
json,
body,
conversationId,
expirationStartTimestamp,
@ -2638,7 +2674,7 @@ async function saveMessage(
) values (
$id,
$json,
$body,
$conversationId,
$expirationStartTimestamp,
@ -2967,7 +3003,7 @@ async function getLastConversationActivity({
const row = await db.get(
`SELECT * FROM messages WHERE
conversationId = $conversationId AND
(type IS NULL
(type IS NULL
OR
type NOT IN (
'profile-change',

16
ts/sql/initialize.ts Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { ipcRenderer as ipc } from 'electron';
import fs from 'fs-extra';
import pify from 'pify';
import sql from './Server';
const getRealPath = pify(fs.realpath);
export async function initialize(): Promise<void> {
const configDir = await getRealPath(ipc.sendSync('get-user-data-path'));
const key = ipc.sendSync('user-config-key');
await sql.initializeRenderer({ configDir, key });
}

5
ts/test-electron/models/messages_test.ts
View file

@ -23,7 +23,10 @@ describe('Message', () => {
function createMessage(attrs: { [key: string]: unknown }) {
const messages = new window.Whisper.MessageCollection();
return messages.add(attrs);
return messages.add({
received_at: Date.now(),
...attrs,
});
}
before(async () => {

11
ts/test-node/components/media-gallery/groupMessagesByDate_test.ts
View file

@ -17,6 +17,7 @@ const toMediaItem = (date: Date): MediaItemType => ({
message: {
id: 'id',
received_at: date.getTime(),
received_at_ms: date.getTime(),
attachments: [],
},
attachment: {
@ -57,6 +58,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1523534400000,
received_at_ms: 1523534400000,
attachments: [],
},
attachment: {
@ -71,6 +73,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1523491260000,
received_at_ms: 1523491260000,
attachments: [],
},
attachment: {
@ -90,6 +93,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1523491140000,
received_at_ms: 1523491140000,
attachments: [],
},
attachment: {
@ -109,6 +113,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1523232060000,
received_at_ms: 1523232060000,
attachments: [],
},
attachment: {
@ -128,6 +133,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1523231940000,
received_at_ms: 1523231940000,
attachments: [],
},
attachment: {
@ -142,6 +148,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1522540860000,
received_at_ms: 1522540860000,
attachments: [],
},
attachment: {
@ -163,6 +170,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1522540740000,
received_at_ms: 1522540740000,
attachments: [],
},
attachment: {
@ -177,6 +185,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1519912800000,
received_at_ms: 1519912800000,
attachments: [],
},
attachment: {
@ -198,6 +207,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1298937540000,
received_at_ms: 1298937540000,
attachments: [],
},
attachment: {
@ -212,6 +222,7 @@ describe('groupMediaItemsByDate', () => {
message: {
id: 'id',
received_at: 1296554400000,
received_at_ms: 1296554400000,
attachments: [],
},
attachment: {

3
ts/textsecure.d.ts vendored
View file

@ -23,6 +23,7 @@ export type UnprocessedType = {
decrypted?: string;
envelope?: string;
id: string;
timestamp: number;
serverTimestamp?: number;
source?: string;
sourceDevice?: number;
@ -795,6 +796,8 @@ export declare class EnvelopeClass {
// Note: these additional properties are added in the course of processing
id: string;
receivedAtCounter: number;
receivedAtDate: number;
unidentifiedDeliveryReceived?: boolean;
messageAgeSec?: number;
}

191
ts/textsecure/MessageReceiver.ts
View file

@ -203,7 +203,7 @@ class MessageReceiverInner extends EventTarget {
this.cacheAddBatcher = createBatcher<CacheAddItemType>({
wait: 200,
maxSize: 30,
processBatch: this.cacheAndQueueBatch.bind(this),
processBatch: this.cacheAndHandleBatch.bind(this),
});
this.cacheUpdateBatcher = createBatcher<CacheUpdateItemType>({
wait: 500,
@ -237,7 +237,7 @@ class MessageReceiverInner extends EventTarget {
}
// We always process our cache before processing a new websocket message
this.pendingQueue.add(async () => this.queueAllCached());
this.pendingQueue.add(async () => this.handleAllCached());
this.count = 0;
if (this.hasConnected) {
@ -428,13 +428,16 @@ class MessageReceiverInner extends EventTarget {
? envelope.serverTimestamp.toNumber()
: null;
envelope.receivedAtCounter = window.Signal.Util.incrementMessageCounter();
envelope.receivedAtDate = Date.now();
// Calculate the message age (time on server).
envelope.messageAgeSec = this.calculateMessageAge(
headers,
envelope.serverTimestamp
);
this.cacheAndQueue(envelope, plaintext, request);
this.cacheAndHandle(envelope, plaintext, request);
} catch (e) {
request.respond(500, 'Bad encrypted websocket message');
window.log.error(
@ -553,16 +556,17 @@ class MessageReceiverInner extends EventTarget {
this.dispatchEvent(ev);
}
async queueAllCached() {
async handleAllCached() {
const items = await this.getAllFromCache();
const max = items.length;
for (let i = 0; i < max; i += 1) {
// eslint-disable-next-line no-await-in-loop
await this.queueCached(items[i]);
await this.handleCachedEnvelope(items[i]);
}
}
async queueCached(item: UnprocessedType) {
async handleCachedEnvelope(item: UnprocessedType) {
window.log.info('MessageReceiver.handleCachedEnvelope', item.id);
try {
let envelopePlaintext: ArrayBuffer;
@ -576,7 +580,7 @@ class MessageReceiverInner extends EventTarget {
);
} else {
throw new Error(
'MessageReceiver.queueCached: item.envelope was malformed'
'MessageReceiver.handleCachedEnvelope: item.envelope was malformed'
);
}
@ -584,6 +588,8 @@ class MessageReceiverInner extends EventTarget {
envelopePlaintext
);
envelope.id = item.id;
envelope.receivedAtCounter = item.timestamp;
envelope.receivedAtDate = Date.now();
envelope.source = envelope.source || item.source;
envelope.sourceUuid = envelope.sourceUuid || item.sourceUuid;
envelope.sourceDevice = envelope.sourceDevice || item.sourceDevice;
@ -605,13 +611,13 @@ class MessageReceiverInner extends EventTarget {
} else {
throw new Error('Cached decrypted value was not a string!');
}
this.queueDecryptedEnvelope(envelope, payloadPlaintext);
this.handleDecryptedEnvelope(envelope, payloadPlaintext);
} else {
this.queueEnvelope(envelope);
this.handleEnvelope(envelope);
}
} catch (error) {
window.log.error(
'queueCached error handling item',
'handleCachedEnvelope error handling item',
item.id,
'removing it. Error:',
error && error.stack ? error.stack : error
@ -622,7 +628,7 @@ class MessageReceiverInner extends EventTarget {
await window.textsecure.storage.unprocessed.remove(id);
} catch (deleteError) {
window.log.error(
'queueCached error deleting item',
'handleCachedEnvelope error deleting item',
item.id,
'Error:',
deleteError && deleteError.stack ? deleteError.stack : deleteError
@ -656,7 +662,7 @@ class MessageReceiverInner extends EventTarget {
if (this.isEmptied) {
this.clearRetryTimeout();
this.retryCachedTimeout = setTimeout(() => {
this.pendingQueue.add(async () => this.queueAllCached());
this.pendingQueue.add(async () => this.handleAllCached());
}, RETRY_TIMEOUT);
}
}
@ -705,7 +711,8 @@ class MessageReceiverInner extends EventTarget {
);
}
async cacheAndQueueBatch(items: Array<CacheAddItemType>) {
async cacheAndHandleBatch(items: Array<CacheAddItemType>) {
window.log.info('MessageReceiver.cacheAndHandleBatch', items.length);
const dataArray = items.map(item => item.data);
try {
await window.textsecure.storage.unprocessed.batchAdd(dataArray);
@ -714,16 +721,16 @@ class MessageReceiverInner extends EventTarget {
item.request.respond(200, 'OK');
} catch (error) {
window.log.error(
'cacheAndQueueBatch: Failed to send 200 to server; still queuing envelope'
'cacheAndHandleBatch: Failed to send 200 to server; still queuing envelope'
);
}
this.queueEnvelope(item.envelope);
this.handleEnvelope(item.envelope);
});
this.maybeScheduleRetryTimeout();
} catch (error) {
window.log.error(
'cacheAndQueue error trying to add messages to cache:',
'cacheAndHandleBatch error trying to add messages to cache:',
error && error.stack ? error.stack : error
);
@ -733,7 +740,7 @@ class MessageReceiverInner extends EventTarget {
}
}
cacheAndQueue(
cacheAndHandle(
envelope: EnvelopeClass,
plaintext: ArrayBuffer,
request: IncomingWebSocketRequest
@ -743,7 +750,7 @@ class MessageReceiverInner extends EventTarget {
id,
version: 2,
envelope: MessageReceiverInner.arrayBufferToStringBase64(plaintext),
timestamp: Date.now(),
timestamp: envelope.receivedAtCounter,
attempts: 1,
};
this.cacheAddBatcher.add({
@ -754,6 +761,7 @@ class MessageReceiverInner extends EventTarget {
}
async cacheUpdateBatch(items: Array<Partial<UnprocessedType>>) {
window.log.info('MessageReceiver.cacheUpdateBatch', items.length);
await window.textsecure.storage.unprocessed.addDecryptedDataToList(items);
}
@ -778,43 +786,71 @@ class MessageReceiverInner extends EventTarget {
this.cacheRemoveBatcher.add(id);
}
async queueDecryptedEnvelope(
// Same as handleEnvelope, just without the decryption step. Necessary for handling
// messages which were successfully decrypted, but application logic didn't finish
// processing.
async handleDecryptedEnvelope(
envelope: EnvelopeClass,
plaintext: ArrayBuffer
) {
): Promise<void> {
const id = this.getEnvelopeId(envelope);
window.log.info('queueing decrypted envelope', id);
window.log.info('MessageReceiver.handleDecryptedEnvelope', id);
const task = this.handleDecryptedEnvelope.bind(this, envelope, plaintext);
const taskWithTimeout = window.textsecure.createTaskWithTimeout(
task,
`queueEncryptedEnvelope ${id}`
);
const promise = this.addToQueue(taskWithTimeout);
try {
if (this.stoppingProcessing) {
return;
}
// No decryption is required for delivery receipts, so the decrypted field of
// the Unprocessed model will never be set
return promise.catch(error => {
if (envelope.content) {
await this.innerHandleContentMessage(envelope, plaintext);
return;
}
if (envelope.legacyMessage) {
await this.innerHandleLegacyMessage(envelope, plaintext);
return;
}
this.removeFromCache(envelope);
throw new Error('Received message with no content and no legacyMessage');
} catch (error) {
window.log.error(
`queueDecryptedEnvelope error handling envelope ${id}:`,
`handleDecryptedEnvelope error handling envelope ${id}:`,
error && error.extra ? JSON.stringify(error.extra) : '',
error && error.stack ? error.stack : error
);
});
}
}
async queueEnvelope(envelope: EnvelopeClass) {
async handleEnvelope(envelope: EnvelopeClass) {
const id = this.getEnvelopeId(envelope);
window.log.info('queueing envelope', id);
window.log.info('MessageReceiver.handleEnvelope', id);
const task = this.handleEnvelope.bind(this, envelope);
const taskWithTimeout = window.textsecure.createTaskWithTimeout(
task,
`queueEnvelope ${id}`
);
const promise = this.addToQueue(taskWithTimeout);
try {
if (this.stoppingProcessing) {
return Promise.resolve();
}
return promise.catch(error => {
if (envelope.type === window.textsecure.protobuf.Envelope.Type.RECEIPT) {
return this.onDeliveryReceipt(envelope);
}
if (envelope.content) {
return this.handleContentMessage(envelope);
}
if (envelope.legacyMessage) {
return this.handleLegacyMessage(envelope);
}
this.removeFromCache(envelope);
throw new Error('Received message with no content and no legacyMessage');
} catch (error) {
const args = [
'queueEnvelope error handling envelope',
'handleEnvelope error handling envelope',
this.getEnvelopeId(envelope),
':',
error && error.extra ? JSON.stringify(error.extra) : '',
@ -825,54 +861,9 @@ class MessageReceiverInner extends EventTarget {
} else {
window.log.error(...args);
}
});
}
// Same as handleEnvelope, just without the decryption step. Necessary for handling
// messages which were successfully decrypted, but application logic didn't finish
// processing.
async handleDecryptedEnvelope(
envelope: EnvelopeClass,
plaintext: ArrayBuffer
): Promise<void> {
if (this.stoppingProcessing) {
return;
}
// No decryption is required for delivery receipts, so the decrypted field of
// the Unprocessed model will never be set
if (envelope.content) {
await this.innerHandleContentMessage(envelope, plaintext);
return;
}
if (envelope.legacyMessage) {
await this.innerHandleLegacyMessage(envelope, plaintext);
return;
}
this.removeFromCache(envelope);
throw new Error('Received message with no content and no legacyMessage');
}
async handleEnvelope(envelope: EnvelopeClass) {
if (this.stoppingProcessing) {
return Promise.resolve();
}
if (envelope.type === window.textsecure.protobuf.Envelope.Type.RECEIPT) {
return this.onDeliveryReceipt(envelope);
}
if (envelope.content) {
return this.handleContentMessage(envelope);
}
if (envelope.legacyMessage) {
return this.handleLegacyMessage(envelope);
}
this.removeFromCache(envelope);
throw new Error('Received message with no content and no legacyMessage');
return undefined;
}
getStatus() {
@ -1257,6 +1248,10 @@ class MessageReceiverInner extends EventTarget {
envelope: EnvelopeClass,
sentContainer: SyncMessageClass.Sent
) {
window.log.info(
'MessageReceiver.handleSentMessage',
this.getEnvelopeId(envelope)
);
const {
destination,
destinationUuid,
@ -1324,6 +1319,8 @@ class MessageReceiverInner extends EventTarget {
unidentifiedStatus,
message,
isRecipientUpdate,
receivedAtCounter: envelope.receivedAtCounter,
receivedAtDate: envelope.receivedAtDate,
};
if (expirationStartTimestamp) {
ev.data.expirationStartTimestamp = expirationStartTimestamp.toNumber();
@ -1334,7 +1331,10 @@ class MessageReceiverInner extends EventTarget {
}
async handleDataMessage(envelope: EnvelopeClass, msg: DataMessageClass) {
window.log.info('data message from', this.getEnvelopeId(envelope));
window.log.info(
'MessageReceiver.handleDataMessage',
this.getEnvelopeId(envelope)
);
let p: Promise<any> = Promise.resolve();
// eslint-disable-next-line no-bitwise
const destination = envelope.sourceUuid || envelope.source;
@ -1412,6 +1412,8 @@ class MessageReceiverInner extends EventTarget {
serverTimestamp: envelope.serverTimestamp,
unidentifiedDeliveryReceived: envelope.unidentifiedDeliveryReceived,
message,
receivedAtCounter: envelope.receivedAtCounter,
receivedAtDate: envelope.receivedAtDate,
};
return this.dispatchAndWait(ev);
})
@ -1419,6 +1421,10 @@ class MessageReceiverInner extends EventTarget {
}
async handleLegacyMessage(envelope: EnvelopeClass) {
window.log.info(
'MessageReceiver.handleLegacyMessage',
this.getEnvelopeId(envelope)
);
return this.decrypt(envelope, envelope.legacyMessage).then(plaintext => {
if (!plaintext) {
window.log.warn('handleLegacyMessage: plaintext was falsey');
@ -1437,6 +1443,10 @@ class MessageReceiverInner extends EventTarget {
}
async handleContentMessage(envelope: EnvelopeClass) {
window.log.info(
'MessageReceiver.handleContentMessage',
this.getEnvelopeId(envelope)
);
return this.decrypt(envelope, envelope.content).then(plaintext => {
if (!plaintext) {
window.log.warn('handleContentMessage: plaintext was falsey');
@ -1579,7 +1589,10 @@ class MessageReceiverInner extends EventTarget {
}
handleNullMessage(envelope: EnvelopeClass) {
window.log.info('null message from', this.getEnvelopeId(envelope));
window.log.info(
'MessageReceiver.handleNullMessage',
this.getEnvelopeId(envelope)
);
this.removeFromCache(envelope);
}
@ -1778,7 +1791,6 @@ class MessageReceiverInner extends EventTarget {
return undefined;
}
if (syncMessage.read && syncMessage.read.length) {
window.log.info('read messages from', this.getEnvelopeId(envelope));
return this.handleRead(envelope, syncMessage.read);
}
if (syncMessage.verified) {
@ -1952,6 +1964,7 @@ class MessageReceiverInner extends EventTarget {
envelope: EnvelopeClass,
read: Array<SyncMessageClass.Read>
) {
window.log.info('MessageReceiver.handleRead', this.getEnvelopeId(envelope));
const results = [];
for (let i = 0; i < read.length; i += 1) {
const ev = new Event('readSync');

8
ts/util/getMessageTimestamp.ts Normal file
View file

@ -0,0 +1,8 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { Message } from '../components/conversation/media-gallery/types/Message';
export function getMessageTimestamp(message: Message): number {
return message.received_at_ms || message.received_at;
}

23
ts/util/incrementMessageCounter.ts Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { debounce } from 'lodash';
export function incrementMessageCounter(): number {
if (!window.receivedAtCounter) {
window.receivedAtCounter =
Number(localStorage.getItem('lastReceivedAtCounter')) || Date.now();
}
window.receivedAtCounter += 1;
debouncedUpdateLastReceivedAt();
return window.receivedAtCounter;
}
const debouncedUpdateLastReceivedAt = debounce(() => {
localStorage.setItem(
'lastReceivedAtCounter',
String(window.receivedAtCounter)
);
}, 500);

9
ts/util/index.ts
View file

@ -14,8 +14,10 @@ import { getStringForProfileChange } from './getStringForProfileChange';
import { getTextWithMentions } from './getTextWithMentions';
import { getUserAgent } from './getUserAgent';
import { hasExpired } from './hasExpired';
import { incrementMessageCounter } from './incrementMessageCounter';
import { isFileDangerous } from './isFileDangerous';
import { makeLookup } from './makeLookup';
import { saveNewMessageBatcher, updateMessageBatcher } from './messageBatcher';
import { missingCaseError } from './missingCaseError';
import { parseRemoteClientExpiration } from './parseRemoteClientExpiration';
import { sleep } from './sleep';
@ -29,6 +31,8 @@ import {
import * as zkgroup from './zkgroup';
export {
GoogleChrome,
Registration,
arrayBufferToObjectURL,
combineNames,
createBatcher,
@ -40,18 +44,19 @@ export {
getStringForProfileChange,
getTextWithMentions,
getUserAgent,
GoogleChrome,
hasExpired,
incrementMessageCounter,
isFileDangerous,
longRunningTaskWrapper,
makeLookup,
mapToSupportLocale,
missingCaseError,
parseRemoteClientExpiration,
Registration,
saveNewMessageBatcher,
sessionRecordToProtobuf,
sessionStructureToArrayBuffer,
sleep,
toWebSafeBase64,
updateMessageBatcher,
zkgroup,
};

4
ts/util/lint/exceptions.json
View file

@ -15022,7 +15022,7 @@
"rule": "React-createRef",
"path": "ts/components/conversation/media-gallery/MediaGallery.js",
"line": " this.focusRef = react_1.default.createRef();",
"lineNumber": 31,
"lineNumber": 32,
"reasonCategory": "usageTrusted",
"updated": "2019-11-01T22:46:33.013Z",
"reasonDetail": "Used for setting focus only"
@ -15031,7 +15031,7 @@
"rule": "React-createRef",
"path": "ts/components/conversation/media-gallery/MediaGallery.tsx",
"line": " public readonly focusRef: React.RefObject<HTMLDivElement> = React.createRef();",
"lineNumber": 71,
"lineNumber": 72,
"reasonCategory": "usageTrusted",
"updated": "2019-11-01T22:46:33.013Z",
"reasonDetail": "Used for setting focus only"

24
ts/util/messageBatcher.ts Normal file
View file

@ -0,0 +1,24 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { MessageAttributesType } from '../model-types.d';
import { createBatcher } from './batcher';
import { createWaitBatcher } from './waitBatcher';
export const updateMessageBatcher = createBatcher<MessageAttributesType>({
wait: 500,
maxSize: 50,
processBatch: async (messages: Array<MessageAttributesType>) => {
window.log.info('updateMessageBatcher', messages.length);
await window.Signal.Data.saveMessages(messages, {});
},
});
export const saveNewMessageBatcher = createWaitBatcher<MessageAttributesType>({
wait: 500,
maxSize: 30,
processBatch: async (messages: Array<MessageAttributesType>) => {
window.log.info('saveNewMessageBatcher', messages.length);
await window.Signal.Data.saveMessages(messages, { forceSave: true });
},
});

7
ts/window.d.ts vendored
View file

@ -137,10 +137,12 @@ declare global {
WhatIsThis: WhatIsThis;
attachmentDownloadQueue: Array<MessageModel>;
baseAttachmentsPath: string;
baseStickersPath: string;
baseTempPath: string;
dcodeIO: DCodeIOType;
receivedAtCounter: number;
enterKeyboardMode: () => void;
enterMouseMode: () => void;
getAccountManager: () => AccountManager | undefined;
@ -246,6 +248,9 @@ declare global {
titleBarDoubleClick: () => void;
unregisterForActive: (handler: () => void) => void;
updateTrayIcon: (count: number) => void;
sqlInitializer: {
initialize: () => Promise<void>;
};
Backbone: typeof Backbone;
Signal: {
@ -561,6 +566,8 @@ export type DCodeIOType = {
};
type MessageControllerType = {
findBySender: (sender: string) => MessageModel | null;
findBySentAt: (sentAt: number) => MessageModel | null;
register: (id: string, model: MessageModel) => MessageModel;
unregister: (id: string) => void;
};