From 3105b7747594c25c1aa8c7d05a5ec32f4c0342c0 Mon Sep 17 00:00:00 2001 From: Scott Nonnenberg Date: Thu, 26 Jul 2018 18:13:56 -0700 Subject: [PATCH] Migrate to SQLCipher for messages/cache Quite a few other fixes, including: - Sending to contact with no avatar yet (not synced from mobile) - Left pane doesn't update quickly or at all on new message - Left pane doesn't show sent or error status Also: - Contributing.md: Ensure set of linux dev dependencies is complete --- .gitignore | 1 + CONTRIBUTING.md | 6 +- app/attachment_channel.js | 36 ++ app/sql.js | 694 +++++++++++++++++++++++++ app/sql_channel.js | 55 ++ js/background.js | 245 +++++---- js/conversation_controller.js | 43 +- js/models/conversations.js | 203 +++----- js/models/messages.js | 200 +++++-- js/modules/backup.js | 6 +- js/modules/data.js | 564 +++++++++++--------- js/modules/migrate_to_sql.js | 167 ++++++ js/modules/settings.js | 15 +- js/modules/signal.js | 4 + js/modules/types/message.js | 38 +- js/signal_protocol_store.js | 18 +- js/views/clear_data_view.js | 10 +- js/views/conversation_view.js | 31 +- libtextsecure/errors.js | 39 +- libtextsecure/message_receiver.js | 14 +- libtextsecure/sendmessage.js | 19 +- main.js | 67 +-- package.json | 2 + stylesheets/_modules.scss | 3 + test/_test.js | 2 + test/keychange_listener_test.js | 22 +- ts/components/ConversationListItem.md | 77 +++ ts/components/ConversationListItem.tsx | 32 +- yarn.lock | 109 +++- 29 files changed, 2006 insertions(+), 716 deletions(-) create mode 100644 app/attachment_channel.js create mode 100644 app/sql.js create mode 100644 app/sql_channel.js create mode 100644 js/modules/migrate_to_sql.js diff --git a/.gitignore b/.gitignore index 1eb148df0..3ee9ea879 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ release/ /dev-app-update.yml .nyc_output/ *.sublime* +sql/ # generated files js/components.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 658c5e20f..25fc239d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,8 +45,10 @@ Then you need `git`, if you don't have that yet: https://git-scm.com/ ### Linux 1. Pick your favorite package manager. -1. Install Python 2.x. -1. Install GCC. +1. Install `python` +1. Install `gcc` +1. Install `g++` +1. Install `make` ### All platforms diff --git a/app/attachment_channel.js b/app/attachment_channel.js new file mode 100644 index 000000000..46a4b9ec8 --- /dev/null +++ b/app/attachment_channel.js @@ -0,0 +1,36 @@ +const electron = require('electron'); +const Attachments = require('./attachments'); +const rimraf = require('rimraf'); + +const { ipcMain } = electron; + +module.exports = { + initialize, +}; + +let initialized = false; + +const ERASE_ATTACHMENTS_KEY = 'erase-attachments'; + +async function initialize({ configDir }) { + if (initialized) { + throw new Error('initialze: Already initialized!'); + } + initialized = true; + + console.log('Ensure attachments directory exists'); + await Attachments.ensureDirectory(configDir); + + const attachmentsDir = Attachments.getPath(configDir); + + ipcMain.on(ERASE_ATTACHMENTS_KEY, async event => { + try { + rimraf.sync(attachmentsDir); + event.sender.send(`${ERASE_ATTACHMENTS_KEY}-done`); + } catch (error) { + const errorForDisplay = error && error.stack ? error.stack : error; + console.log(`sql-erase error: ${errorForDisplay}`); + event.sender.send(`${ERASE_ATTACHMENTS_KEY}-done`, error); + } + }); +} diff --git a/app/sql.js b/app/sql.js new file mode 100644 index 000000000..000230c86 --- /dev/null +++ b/app/sql.js @@ -0,0 +1,694 @@ +const path = require('path'); +const mkdirp = require('mkdirp'); +const rimraf = require('rimraf'); +const sql = require('@journeyapps/sqlcipher'); +const pify = require('pify'); +const uuidv4 = require('uuid/v4'); +const { map, isString } = require('lodash'); + +// To get long stack traces +// https://github.com/mapbox/node-sqlite3/wiki/API#sqlite3verbose +sql.verbose(); + +module.exports = { + initialize, + close, + removeDB, + + saveMessage, + saveMessages, + removeMessage, + getUnreadByConversation, + getMessageBySender, + getMessageById, + getAllMessageIds, + getMessagesBySentAt, + getExpiredMessages, + getNextExpiringMessage, + getMessagesByConversation, + + getAllUnprocessed, + saveUnprocessed, + getUnprocessedById, + saveUnprocesseds, + removeUnprocessed, + removeAllUnprocessed, + + removeAll, + + getMessagesNeedingUpgrade, + getMessagesWithVisualMediaAttachments, + getMessagesWithFileAttachments, +}; + +function generateUUID() { + return uuidv4(); +} + +function objectToJSON(data) { + return JSON.stringify(data); +} +function jsonToObject(json) { + return JSON.parse(json); +} + +async function openDatabase(filePath) { + return new Promise((resolve, reject) => { + const instance = new sql.Database(filePath, error => { + if (error) { + return reject(error); + } + + return resolve(instance); + }); + }); +} + +function promisify(rawInstance) { + /* eslint-disable no-param-reassign */ + rawInstance.close = pify(rawInstance.close.bind(rawInstance)); + rawInstance.run = pify(rawInstance.run.bind(rawInstance)); + rawInstance.get = pify(rawInstance.get.bind(rawInstance)); + rawInstance.all = pify(rawInstance.all.bind(rawInstance)); + rawInstance.each = pify(rawInstance.each.bind(rawInstance)); + rawInstance.exec = pify(rawInstance.exec.bind(rawInstance)); + rawInstance.prepare = pify(rawInstance.prepare.bind(rawInstance)); + /* eslint-enable */ + + return rawInstance; +} + +async function getSQLiteVersion(instance) { + const row = await instance.get('select sqlite_version() AS sqlite_version'); + return row.sqlite_version; +} + +async function getSchemaVersion(instance) { + const row = await instance.get('PRAGMA schema_version;'); + return row.schema_version; +} + +async function getSQLCipherVersion(instance) { + const row = await instance.get('PRAGMA cipher_version;'); + try { + return row.cipher_version; + } catch (e) { + return null; + } +} + +const INVALID_KEY = /[^0-9A-Fa-f]/; +async function setupSQLCipher(instance, { key }) { + const match = INVALID_KEY.exec(key); + if (match) { + throw new Error(`setupSQLCipher: key '${key}' is not valid`); + } + + // https://www.zetetic.net/sqlcipher/sqlcipher-api/#key + await instance.run(`PRAGMA key = "x'${key}'";`); +} + +async function updateToSchemaVersion1(currentVersion, instance) { + if (currentVersion >= 1) { + return; + } + + console.log('updateToSchemaVersion1: starting...'); + + await instance.run('BEGIN TRANSACTION;'); + + await instance.run( + `CREATE TABLE messages( + id STRING PRIMARY KEY ASC, + json TEXT, + + unread INTEGER, + expires_at INTEGER, + sent_at INTEGER, + schemaVersion INTEGER, + conversationId STRING, + received_at INTEGER, + source STRING, + sourceDevice STRING, + hasAttachments INTEGER, + hasFileAttachments INTEGER, + hasVisualMediaAttachments INTEGER + );` + ); + + await instance.run(`CREATE INDEX messages_unread ON messages ( + unread + );`); + await instance.run(`CREATE INDEX messages_expires_at ON messages ( + expires_at + );`); + await instance.run(`CREATE INDEX messages_receipt ON messages ( + sent_at + );`); + await instance.run(`CREATE INDEX messages_schemaVersion ON messages ( + schemaVersion + );`); + + await instance.run(`CREATE INDEX messages_conversation ON messages ( + conversationId, + received_at + );`); + + await instance.run(`CREATE INDEX messages_duplicate_check ON messages ( + source, + sourceDevice, + sent_at + );`); + await instance.run(`CREATE INDEX messages_hasAttachments ON messages ( + conversationId, + hasAttachments, + received_at + );`); + await instance.run(`CREATE INDEX messages_hasFileAttachments ON messages ( + conversationId, + hasFileAttachments, + received_at + );`); + await instance.run(`CREATE INDEX messages_hasVisualMediaAttachments ON messages ( + conversationId, + hasVisualMediaAttachments, + received_at + );`); + + await instance.run(`CREATE TABLE unprocessed( + id STRING, + timestamp INTEGER, + json TEXT + );`); + await instance.run(`CREATE INDEX unprocessed_id ON unprocessed ( + id + );`); + await instance.run(`CREATE INDEX unprocessed_timestamp ON unprocessed ( + timestamp + );`); + + await instance.run('PRAGMA schema_version = 1;'); + await instance.run('COMMIT TRANSACTION;'); + + console.log('updateToSchemaVersion1: success!'); +} + +const SCHEMA_VERSIONS = [updateToSchemaVersion1]; + +async function updateSchema(instance) { + const sqliteVersion = await getSQLiteVersion(instance); + const schemaVersion = await getSchemaVersion(instance); + const cipherVersion = await getSQLCipherVersion(instance); + console.log( + 'updateSchema:', + `Current schema version: ${schemaVersion};`, + `Most recent schema version: ${SCHEMA_VERSIONS.length};`, + `SQLite version: ${sqliteVersion};`, + `SQLCipher version: ${cipherVersion};` + ); + + for (let index = 0, max = SCHEMA_VERSIONS.length; index < max; index += 1) { + const runSchemaUpdate = SCHEMA_VERSIONS[index]; + + // Yes, we really want to do this asynchronously, in order + // eslint-disable-next-line no-await-in-loop + await runSchemaUpdate(schemaVersion, instance); + } +} + +let db; +let filePath; + +async function initialize({ configDir, key }) { + if (db) { + throw new Error('Cannot initialize more than once!'); + } + + if (!isString(configDir)) { + throw new Error('initialize: configDir is required!'); + } + if (!isString(key)) { + throw new Error('initialize: key` is required!'); + } + + const dbDir = path.join(configDir, 'sql'); + mkdirp.sync(dbDir); + + filePath = path.join(dbDir, 'db.sqlite'); + const sqlInstance = await openDatabase(filePath); + const promisified = promisify(sqlInstance); + + // promisified.on('trace', statement => console._log(statement)); + + await setupSQLCipher(promisified, { key }); + await updateSchema(promisified); + + db = promisified; +} + +async function close() { + const dbRef = db; + db = null; + await dbRef.close(); +} + +async function removeDB() { + if (db) { + throw new Error('removeDB: Cannot erase database when it is open!'); + } + + rimraf.sync(filePath); +} + +async function saveMessage(data, { forceSave } = {}) { + const { + conversationId, + // eslint-disable-next-line camelcase + expires_at, + hasAttachments, + hasFileAttachments, + hasVisualMediaAttachments, + id, + // eslint-disable-next-line camelcase + received_at, + schemaVersion, + // eslint-disable-next-line camelcase + sent_at, + source, + sourceDevice, + unread, + } = data; + + if (id && !forceSave) { + await db.run( + `UPDATE messages SET + json = $json, + conversationId = $conversationId, + expires_at = $expires_at, + hasAttachments = $hasAttachments, + hasFileAttachments = $hasFileAttachments, + hasVisualMediaAttachments = $hasVisualMediaAttachments, + id = $id, + received_at = $received_at, + schemaVersion = $schemaVersion, + sent_at = $sent_at, + source = $source, + sourceDevice = $sourceDevice, + unread = $unread + WHERE id = $id;`, + { + $id: id, + $json: objectToJSON(data), + + $conversationId: conversationId, + $expires_at: expires_at, + $hasAttachments: hasAttachments, + $hasFileAttachments: hasFileAttachments, + $hasVisualMediaAttachments: hasVisualMediaAttachments, + $received_at: received_at, + $schemaVersion: schemaVersion, + $sent_at: sent_at, + $source: source, + $sourceDevice: sourceDevice, + $unread: unread, + } + ); + + return id; + } + + const toCreate = { + ...data, + id: id || generateUUID(), + }; + + await db.run( + `INSERT INTO messages ( + id, + json, + + conversationId, + expires_at, + hasAttachments, + hasFileAttachments, + hasVisualMediaAttachments, + received_at, + schemaVersion, + sent_at, + source, + sourceDevice, + unread + ) values ( + $id, + $json, + + $conversationId, + $expires_at, + $hasAttachments, + $hasFileAttachments, + $hasVisualMediaAttachments, + $received_at, + $schemaVersion, + $sent_at, + $source, + $sourceDevice, + $unread + );`, + { + $id: toCreate.id, + $json: objectToJSON(toCreate), + + $conversationId: conversationId, + $expires_at: expires_at, + $hasAttachments: hasAttachments, + $hasFileAttachments: hasFileAttachments, + $hasVisualMediaAttachments: hasVisualMediaAttachments, + $received_at: received_at, + $schemaVersion: schemaVersion, + $sent_at: sent_at, + $source: source, + $sourceDevice: sourceDevice, + $unread: unread, + } + ); + + return toCreate.id; +} + +async function saveMessages(arrayOfMessages, { forceSave } = {}) { + await Promise.all([ + db.run('BEGIN TRANSACTION;'), + ...map(arrayOfMessages, message => saveMessage(message, { forceSave })), + db.run('COMMIT TRANSACTION;'), + ]); +} + +async function removeMessage(id) { + if (!Array.isArray(id)) { + await db.run('DELETE FROM messages WHERE id = $id;', { $id: id }); + return; + } + + if (!id.length) { + throw new Error('removeMessages: No ids to delete!'); + } + + // Our node interface doesn't seem to allow you to replace one single ? with an array + await db.run( + `DELETE FROM messages WHERE id IN ( ${id.map(() => '?').join(', ')} );`, + id + ); +} + +async function getMessageById(id) { + const row = await db.get('SELECT * FROM messages WHERE id = $id;', { + $id: id, + }); + + if (!row) { + return null; + } + + return jsonToObject(row.json); +} + +async function getAllMessageIds() { + const rows = await db.all('SELECT id FROM messages ORDER BY id ASC;'); + return map(rows, row => row.id); +} + +// eslint-disable-next-line camelcase +async function getMessageBySender({ source, sourceDevice, sent_at }) { + const rows = db.all( + `SELECT json FROM messages WHERE + source = $source AND + sourceDevice = $sourceDevice AND + sent_at = $sent_at;`, + { + $source: source, + $sourceDevice: sourceDevice, + $sent_at: sent_at, + } + ); + + return map(rows, row => jsonToObject(row.json)); +} + +async function getUnreadByConversation(conversationId) { + const rows = await db.all( + `SELECT json FROM messages WHERE + conversationId = $conversationId AND + unread = $unread + ORDER BY received_at DESC;`, + { + $conversationId: conversationId, + $unread: 1, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function getMessagesByConversation( + conversationId, + { limit = 100, receivedAt = Number.MAX_VALUE } = {} +) { + const rows = await db.all( + `SELECT json FROM messages WHERE + conversationId = $conversationId AND + received_at < $received_at + ORDER BY received_at DESC + LIMIT $limit;`, + { + $conversationId: conversationId, + $received_at: receivedAt, + $limit: limit, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function getMessagesBySentAt(sentAt) { + const rows = await db.all( + `SELECT * FROM messages + WHERE sent_at = $sent_at + ORDER BY received_at DESC;`, + { + $sent_at: sentAt, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function getExpiredMessages() { + const now = Date.now(); + + const rows = await db.all( + `SELECT json FROM messages WHERE + expires_at IS NOT NULL AND + expires_at <= $expires_at + ORDER BY expires_at ASC;`, + { + $expires_at: now, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function getNextExpiringMessage() { + const rows = await db.all(` + SELECT json FROM messages + WHERE expires_at IS NOT NULL + ORDER BY expires_at ASC + LIMIT 1; + `); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function saveUnprocessed(data, { forceSave } = {}) { + const { id, timestamp } = data; + + if (forceSave) { + await db.run( + `INSERT INTO unprocessed ( + id, + timestamp, + json + ) values ( + $id, + $timestamp, + $json + );`, + { + $id: id, + $timestamp: timestamp, + $json: objectToJSON(data), + } + ); + + return id; + } + + await db.run( + `UPDATE unprocessed SET + json = $json, + timestamp = $timestamp + WHERE id = $id;`, + { + $id: id, + $timestamp: timestamp, + $json: objectToJSON(data), + } + ); + + return id; +} + +async function saveUnprocesseds(arrayOfUnprocessed, { forceSave } = {}) { + await Promise.all([ + db.run('BEGIN TRANSACTION;'), + ...map(arrayOfUnprocessed, unprocessed => + saveUnprocessed(unprocessed, { forceSave }) + ), + db.run('COMMIT TRANSACTION;'), + ]); +} + +async function getUnprocessedById(id) { + const row = await db.get('SELECT json FROM unprocessed WHERE id = $id;', { + $id: id, + }); + + if (!row) { + return null; + } + + return jsonToObject(row.json); +} + +async function getAllUnprocessed() { + const rows = await db.all( + 'SELECT json FROM unprocessed ORDER BY timestamp ASC;' + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function removeUnprocessed(id) { + if (!Array.isArray(id)) { + await db.run('DELETE FROM unprocessed WHERE id = $id;', { $id: id }); + return; + } + + if (!id.length) { + throw new Error('removeUnprocessed: No ids to delete!'); + } + + // Our node interface doesn't seem to allow you to replace one single ? with an array + await db.run( + `DELETE FROM unprocessed WHERE id IN ( ${id.map(() => '?').join(', ')} );`, + id + ); +} + +async function removeAllUnprocessed() { + await db.run('DELETE FROM unprocessed;'); +} + +async function removeAll() { + await Promise.all([ + db.run('BEGIN TRANSACTION;'), + db.run('DELETE FROM messages;'), + db.run('DELETE FROM unprocessed;'), + db.run('COMMIT TRANSACTION;'), + ]); +} + +async function getMessagesNeedingUpgrade(limit, { maxVersion }) { + const rows = await db.all( + `SELECT json FROM messages + WHERE schemaVersion IS NOT $maxVersion + LIMIT $limit;`, + { + $maxVersion: maxVersion, + $limit: limit, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function getMessagesWithVisualMediaAttachments( + conversationId, + { limit } +) { + const rows = await db.all( + `SELECT json FROM messages WHERE + conversationId = $conversationId AND + hasVisualMediaAttachments = 1 + ORDER BY received_at DESC + LIMIT $limit;`, + { + $conversationId: conversationId, + $limit: limit, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} + +async function getMessagesWithFileAttachments(conversationId, { limit }) { + const rows = await db.all( + `SELECT json FROM messages WHERE + conversationId = $conversationId AND + hasFileAttachments = 1 + ORDER BY received_at DESC + LIMIT $limit;`, + { + $conversationId: conversationId, + $limit: limit, + } + ); + + if (!rows) { + return null; + } + + return map(rows, row => jsonToObject(row.json)); +} diff --git a/app/sql_channel.js b/app/sql_channel.js new file mode 100644 index 000000000..c375b222c --- /dev/null +++ b/app/sql_channel.js @@ -0,0 +1,55 @@ +const electron = require('electron'); +const sql = require('./sql'); + +const { ipcMain } = electron; + +module.exports = { + initialize, +}; + +let initialized = false; + +const SQL_CHANNEL_KEY = 'sql-channel'; +const ERASE_SQL_KEY = 'erase-sql-key'; + +function initialize({ userConfig }) { + if (initialized) { + throw new Error('sqlChannels: already initialized!'); + } + initialized = true; + + if (!userConfig) { + throw new Error('initialize: userConfig is required!'); + } + + ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => { + try { + const fn = sql[callName]; + if (!fn) { + throw new Error( + `sql channel: ${callName} is not an available function` + ); + } + + const result = await fn(...args); + event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result); + } catch (error) { + const errorForDisplay = error && error.stack ? error.stack : error; + console.log( + `sql channel error with call ${callName}: ${errorForDisplay}` + ); + event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, errorForDisplay); + } + }); + + ipcMain.on(ERASE_SQL_KEY, async event => { + try { + userConfig.set('key', null); + event.sender.send(`${ERASE_SQL_KEY}-done`); + } catch (error) { + const errorForDisplay = error && error.stack ? error.stack : error; + console.log(`sql-erase error: ${errorForDisplay}`); + event.sender.send(`${ERASE_SQL_KEY}-done`, error); + } + }); +} diff --git a/js/background.js b/js/background.js index 3d4418fbb..d6e18405f 100644 --- a/js/background.js +++ b/js/background.js @@ -119,7 +119,10 @@ window.log.info('background page reloaded'); window.log.info('environment:', window.getEnvironment()); + let idleDetector; let initialLoadComplete = false; + let newVersion = false; + window.owsDesktopApp = {}; window.document.title = window.getTitle(); @@ -165,85 +168,6 @@ window.log.info('Storage fetch'); storage.fetch(); - const MINIMUM_VERSION = 7; - - async function upgradeMessages() { - const NUM_MESSAGES_PER_BATCH = 10; - window.log.info( - 'upgradeMessages: Mandatory message schema upgrade started.', - `Target version: ${MINIMUM_VERSION}` - ); - - let isMigrationWithoutIndexComplete = false; - while (!isMigrationWithoutIndexComplete) { - const database = Migrations0DatabaseWithAttachmentData.getDatabase(); - // eslint-disable-next-line no-await-in-loop - const batchWithoutIndex = await MessageDataMigrator.processNextBatchWithoutIndex( - { - databaseName: database.name, - minDatabaseVersion: database.version, - numMessagesPerBatch: NUM_MESSAGES_PER_BATCH, - upgradeMessageSchema, - maxVersion: MINIMUM_VERSION, - BackboneMessage: Whisper.Message, - saveMessage: window.Signal.Data.saveMessage, - } - ); - window.log.info( - 'upgradeMessages: upgrade without index', - batchWithoutIndex - ); - isMigrationWithoutIndexComplete = batchWithoutIndex.done; - } - window.log.info('upgradeMessages: upgrade without index complete!'); - - let isMigrationWithIndexComplete = false; - while (!isMigrationWithIndexComplete) { - // eslint-disable-next-line no-await-in-loop - const batchWithIndex = await MessageDataMigrator.processNext({ - BackboneMessage: Whisper.Message, - BackboneMessageCollection: Whisper.MessageCollection, - numMessagesPerBatch: NUM_MESSAGES_PER_BATCH, - upgradeMessageSchema, - getMessagesNeedingUpgrade: window.Signal.Data.getMessagesNeedingUpgrade, - saveMessage: window.Signal.Data.saveMessage, - maxVersion: MINIMUM_VERSION, - }); - window.log.info('upgradeMessages: upgrade with index', batchWithIndex); - isMigrationWithIndexComplete = batchWithIndex.done; - } - window.log.info('upgradeMessages: upgrade with index complete!'); - - window.log.info('upgradeMessages: Message schema upgrade complete'); - } - - await upgradeMessages(); - - const idleDetector = new IdleDetector(); - let isMigrationWithIndexComplete = false; - window.log.info('Starting background data migration. Target version: latest'); - idleDetector.on('idle', async () => { - const NUM_MESSAGES_PER_BATCH = 1; - - if (!isMigrationWithIndexComplete) { - const batchWithIndex = await MessageDataMigrator.processNext({ - BackboneMessage: Whisper.Message, - BackboneMessageCollection: Whisper.MessageCollection, - numMessagesPerBatch: NUM_MESSAGES_PER_BATCH, - upgradeMessageSchema, - getMessagesNeedingUpgrade: window.Signal.Data.getMessagesNeedingUpgrade, - saveMessage: window.Signal.Data.saveMessage, - }); - window.log.info('Upgrade message schema (with index):', batchWithIndex); - isMigrationWithIndexComplete = batchWithIndex.done; - } - - if (isMigrationWithIndexComplete) { - window.log.info('Background migration complete. Stopping idle detector.'); - idleDetector.stop(); - } - }); - function mapOldThemeToNew(theme) { switch (theme) { case 'dark': @@ -267,6 +191,121 @@ } first = false; + const currentVersion = window.getVersion(); + const lastVersion = storage.get('version'); + newVersion = !lastVersion || currentVersion !== lastVersion; + await storage.put('version', currentVersion); + + if (newVersion) { + if ( + lastVersion && + window.isBeforeVersion(lastVersion, 'v1.15.0-beta.5') + ) { + await window.Signal.Logs.deleteAll(); + window.restart(); + } + + window.log.info( + `New version detected: ${currentVersion}; previous: ${lastVersion}` + ); + } + + const MINIMUM_VERSION = 7; + async function upgradeMessages() { + const NUM_MESSAGES_PER_BATCH = 10; + window.log.info( + 'upgradeMessages: Mandatory message schema upgrade started.', + `Target version: ${MINIMUM_VERSION}` + ); + + let isMigrationWithoutIndexComplete = false; + while (!isMigrationWithoutIndexComplete) { + const database = Migrations0DatabaseWithAttachmentData.getDatabase(); + // eslint-disable-next-line no-await-in-loop + const batchWithoutIndex = await MessageDataMigrator.processNextBatchWithoutIndex( + { + databaseName: database.name, + minDatabaseVersion: database.version, + numMessagesPerBatch: NUM_MESSAGES_PER_BATCH, + upgradeMessageSchema, + maxVersion: MINIMUM_VERSION, + BackboneMessage: Whisper.Message, + saveMessage: window.Signal.Data.saveMessage, + } + ); + window.log.info( + 'upgradeMessages: upgrade without index', + batchWithoutIndex + ); + isMigrationWithoutIndexComplete = batchWithoutIndex.done; + } + window.log.info('upgradeMessages: upgrade without index complete!'); + + let isMigrationWithIndexComplete = false; + while (!isMigrationWithIndexComplete) { + // eslint-disable-next-line no-await-in-loop + const batchWithIndex = await MessageDataMigrator.processNext({ + BackboneMessage: Whisper.Message, + BackboneMessageCollection: Whisper.MessageCollection, + numMessagesPerBatch: NUM_MESSAGES_PER_BATCH, + upgradeMessageSchema, + getMessagesNeedingUpgrade: + window.Signal.Data.getLegacyMessagesNeedingUpgrade, + saveMessage: window.Signal.Data.saveMessage, + maxVersion: MINIMUM_VERSION, + }); + window.log.info('upgradeMessages: upgrade with index', batchWithIndex); + isMigrationWithIndexComplete = batchWithIndex.done; + } + window.log.info('upgradeMessages: upgrade with index complete!'); + + window.log.info('upgradeMessages: Message schema upgrade complete'); + } + + await upgradeMessages(); + + idleDetector = new IdleDetector(); + let isMigrationWithIndexComplete = false; + window.log.info( + `Starting background data migration. Target version: ${ + Message.CURRENT_SCHEMA_VERSION + }` + ); + idleDetector.on('idle', async () => { + const NUM_MESSAGES_PER_BATCH = 1; + + if (!isMigrationWithIndexComplete) { + const batchWithIndex = await MessageDataMigrator.processNext({ + BackboneMessage: Whisper.Message, + BackboneMessageCollection: Whisper.MessageCollection, + numMessagesPerBatch: NUM_MESSAGES_PER_BATCH, + upgradeMessageSchema, + getMessagesNeedingUpgrade: + window.Signal.Data.getMessagesNeedingUpgrade, + saveMessage: window.Signal.Data.saveMessage, + }); + window.log.info('Upgrade message schema (with index):', batchWithIndex); + isMigrationWithIndexComplete = batchWithIndex.done; + } + + if (isMigrationWithIndexComplete) { + window.log.info( + 'Background migration complete. Stopping idle detector.' + ); + idleDetector.stop(); + } + }); + + const db = await Whisper.Database.open(); + await window.Signal.migrateToSQL({ + db, + clearStores: Whisper.Database.clearStores, + handleDOMException: Whisper.Database.handleDOMException, + }); + + // Note: We are not invoking the second set of IndexedDB migrations because it is + // likely that any future migrations will simply extracting things from IndexedDB. + // These make key operations available to IPC handlers created in preload.js window.Events = { getDeviceName: () => textsecure.storage.user.getDeviceName(), @@ -340,14 +379,20 @@ try { await ConversationController.load(); + } catch (error) { + window.log.error( + 'background.js: ConversationController failed to load:', + error && error.stack ? error.stack : error + ); } finally { start(); } }); Whisper.events.on('shutdown', async () => { - idleDetector.stop(); - + if (idleDetector) { + idleDetector.stop(); + } if (messageReceiver) { await messageReceiver.close(); } @@ -376,25 +421,6 @@ }); async function start() { - const currentVersion = window.getVersion(); - const lastVersion = storage.get('version'); - const newVersion = !lastVersion || currentVersion !== lastVersion; - await storage.put('version', currentVersion); - - if (newVersion) { - if ( - lastVersion && - window.isBeforeVersion(lastVersion, 'v1.15.0-beta.5') - ) { - await window.Signal.Logs.deleteAll(); - window.restart(); - } - - window.log.info( - `New version detected: ${currentVersion}; previous: ${lastVersion}` - ); - } - window.dispatchEvent(new Event('storage_ready')); window.log.info('listening for registration events'); @@ -646,15 +672,6 @@ } storage.onready(async () => { - const shouldSkipAttachmentMigrationForNewUsers = firstRun === true; - if (shouldSkipAttachmentMigrationForNewUsers) { - const database = Migrations0DatabaseWithAttachmentData.getDatabase(); - const connection = await Signal.Database.open( - database.name, - database.version - ); - await Signal.Settings.markAttachmentMigrationComplete(connection); - } idleDetector.start(); }); } @@ -1009,8 +1026,14 @@ // These two are important to ensure we don't rip through every message // in the database attempting to upgrade it after starting up again. - textsecure.storage.put(LAST_PROCESSED_INDEX_KEY, lastProcessedIndex); - textsecure.storage.put(IS_MIGRATION_COMPLETE_KEY, isMigrationComplete); + textsecure.storage.put( + IS_MIGRATION_COMPLETE_KEY, + isMigrationComplete || false + ); + textsecure.storage.put( + LAST_PROCESSED_INDEX_KEY, + lastProcessedIndex || null + ); window.log.info('Successfully cleared local configuration'); } catch (eraseError) { diff --git a/js/conversation_controller.js b/js/conversation_controller.js index 75a53f7b3..bf486c8ed 100644 --- a/js/conversation_controller.js +++ b/js/conversation_controller.js @@ -1,4 +1,4 @@ -/* global _, Whisper, Backbone, storage */ +/* global _, Whisper, Backbone, storage, wrapDeferred */ /* eslint-disable more/no-then */ @@ -181,27 +181,34 @@ }, reset() { this._initialPromise = Promise.resolve(); + this._initialFetchComplete = false; conversations.reset([]); }, - load() { + async load() { window.log.info('ConversationController: starting initial fetch'); - this._initialPromise = new Promise((resolve, reject) => { - conversations.fetch().then( - () => { - window.log.info('ConversationController: done with initial fetch'); - this._initialFetchComplete = true; - resolve(); - }, - error => { - window.log.error( - 'ConversationController: initial fetch failed', - error && error.stack ? error.stack : error - ); - reject(error); - } - ); - }); + if (conversations.length) { + throw new Error('ConversationController: Already loaded!'); + } + + const load = async () => { + try { + await wrapDeferred(conversations.fetch()); + this._initialFetchComplete = true; + await Promise.all( + conversations.map(conversation => conversation.updateLastMessage()) + ); + window.log.info('ConversationController: done with initial fetch'); + } catch (error) { + window.log.error( + 'ConversationController: initial fetch failed', + error && error.stack ? error.stack : error + ); + throw error; + } + }; + + this._initialPromise = load(); return this._initialPromise; }, diff --git a/js/models/conversations.js b/js/models/conversations.js index 183610141..d0fb93012 100644 --- a/js/models/conversations.js +++ b/js/models/conversations.js @@ -116,15 +116,21 @@ const debouncedUpdateLastMessage = _.debounce( this.updateLastMessage.bind(this), - 1000 + 200 ); this.listenTo( this.messageCollection, - 'add remove', + 'add remove destroy', debouncedUpdateLastMessage ); - this.listenTo(this.model, 'newmessage', debouncedUpdateLastMessage); + this.listenTo(this.messageCollection, 'sent', this.updateLastMessage); + this.listenTo( + this.messageCollection, + 'send-error', + this.updateLastMessage + ); + this.on('newmessage', this.updateLastMessage); this.on('change:avatar', this.updateAvatarUrl); this.on('change:profileAvatar', this.updateAvatarUrl); this.on('change:profileKey', this.onChangeProfileKey); @@ -133,10 +139,7 @@ // Listening for out-of-band data updates this.on('delivered', this.updateAndMerge); this.on('read', this.updateAndMerge); - this.on('sent', this.updateLastMessage); this.on('expired', this.onExpired); - - this.updateLastMessage(); }, isMe() { @@ -378,98 +381,6 @@ return Promise.all(promises).then(() => lookup); }, - replay(error, message) { - const replayable = new textsecure.ReplayableError(error); - return replayable.replay(message.attributes).catch(e => { - window.log.error('replay error:', e && e.stack ? e.stack : e); - }); - }, - decryptOldIncomingKeyErrors() { - // We want to run just once per conversation - if (this.get('decryptedOldIncomingKeyErrors')) { - return Promise.resolve(); - } - window.log.info( - 'decryptOldIncomingKeyErrors start for', - this.idForLogging() - ); - - const messages = this.messageCollection.filter(message => { - const errors = message.get('errors'); - if (!errors || !errors[0]) { - return false; - } - const error = _.find( - errors, - e => e.name === 'IncomingIdentityKeyError' - ); - - return Boolean(error); - }); - - const markComplete = () => { - window.log.info( - 'decryptOldIncomingKeyErrors complete for', - this.idForLogging() - ); - return new Promise(resolve => { - this.save({ decryptedOldIncomingKeyErrors: true }).always(resolve); - }); - }; - - if (!messages.length) { - return markComplete(); - } - - window.log.info( - 'decryptOldIncomingKeyErrors found', - messages.length, - 'messages to process' - ); - const safeDelete = async message => { - try { - window.Signal.Data.removeMessage(message.id, { - Message: Whisper.Message, - }); - } catch (error) { - // nothing - } - }; - - const promise = this.getIdentityKeys(); - return promise - .then(lookup => - Promise.all( - _.map(messages, message => { - const source = message.get('source'); - const error = _.find( - message.get('errors'), - e => e.name === 'IncomingIdentityKeyError' - ); - - const key = lookup[source]; - if (!key) { - return Promise.resolve(); - } - - if (constantTimeEqualArrayBuffers(key, error.identityKey)) { - return this.replay(error, message).then(() => - safeDelete(message) - ); - } - - return Promise.resolve(); - }) - ) - ) - .catch(error => { - window.log.error( - 'decryptOldIncomingKeyErrors error:', - error && error.stack ? error.stack : error - ); - }) - .then(markComplete); - }, isVerified() { if (this.isPrivate()) { return this.get('verified') === this.verifiedEnum.VERIFIED; @@ -926,12 +837,8 @@ this.id, { limit: 1, MessageCollection: Whisper.MessageCollection } ); - if (!messages.length) { - return; - } const lastMessageModel = messages.at(0); - const lastMessageJSON = lastMessageModel ? lastMessageModel.toJSON() : null; @@ -968,7 +875,7 @@ } }, - updateExpirationTimer( + async updateExpirationTimer( providedExpireTimer, providedSource, receivedAt, @@ -1024,46 +931,48 @@ message.set({ recipients: this.getRecipients() }); } - return Promise.all([ - window.Signal.Data.saveMessage(message.attributes, { - Message: Whisper.Message, - }), - wrapDeferred(this.save({ expireTimer })), - ]).then(() => { - // if change was made remotely, don't send it to the number/group - if (receivedAt) { - return message; - } - - let sendFunc; - if (this.get('type') === 'private') { - sendFunc = textsecure.messaging.sendExpirationTimerUpdateToNumber; - } else { - sendFunc = textsecure.messaging.sendExpirationTimerUpdateToGroup; - } - let profileKey; - if (this.get('profileSharing')) { - profileKey = storage.get('profileKey'); - } - const promise = sendFunc( - this.get('id'), - this.get('expireTimer'), - message.get('sent_at'), - profileKey - ); - - return message.send(promise).then(() => message); + const id = await window.Signal.Data.saveMessage(message.attributes, { + Message: Whisper.Message, }); + message.set({ id }); + + await wrapDeferred(this.save({ expireTimer })); + + // if change was made remotely, don't send it to the number/group + if (receivedAt) { + return message; + } + + let sendFunc; + if (this.get('type') === 'private') { + sendFunc = textsecure.messaging.sendExpirationTimerUpdateToNumber; + } else { + sendFunc = textsecure.messaging.sendExpirationTimerUpdateToGroup; + } + let profileKey; + if (this.get('profileSharing')) { + profileKey = storage.get('profileKey'); + } + const promise = sendFunc( + this.get('id'), + this.get('expireTimer'), + message.get('sent_at'), + profileKey + ); + + await message.send(promise); + + return message; }, isSearchable() { return !this.get('left') || !!this.get('lastMessage'); }, - endSession() { + async endSession() { if (this.isPrivate()) { const now = Date.now(); - const message = this.messageCollection.create({ + const message = this.messageCollection.add({ conversationId: this.id, type: 'outgoing', sent_at: now, @@ -1072,11 +981,17 @@ recipients: this.getRecipients(), flags: textsecure.protobuf.DataMessage.Flags.END_SESSION, }); + + const id = await window.Signal.Data.saveMessage(message.attributes, { + Message: Whisper.Message, + }); + message.set({ id }); + message.send(textsecure.messaging.resetSession(this.id, now)); } }, - updateGroup(providedGroupUpdate) { + async updateGroup(providedGroupUpdate) { let groupUpdate = providedGroupUpdate; if (this.isPrivate()) { @@ -1086,13 +1001,19 @@ groupUpdate = this.pick(['name', 'avatar', 'members']); } const now = Date.now(); - const message = this.messageCollection.create({ + const message = this.messageCollection.add({ conversationId: this.id, type: 'outgoing', sent_at: now, received_at: now, group_update: groupUpdate, }); + + const id = await window.Signal.Data.saveMessage(message.attributes, { + Message: Whisper.Message, + }); + message.set({ id }); + message.send( textsecure.messaging.updateGroup( this.id, @@ -1103,17 +1024,23 @@ ); }, - leaveGroup() { + async leaveGroup() { const now = Date.now(); if (this.get('type') === 'group') { this.save({ left: true }); - const message = this.messageCollection.create({ + const message = this.messageCollection.add({ group_update: { left: 'You' }, conversationId: this.id, type: 'outgoing', sent_at: now, received_at: now, }); + + const id = await window.Signal.Data.saveMessage(message.attributes, { + Message: Whisper.Message, + }); + message.set({ id }); + message.send(textsecure.messaging.leaveGroup(this.id)); } }, diff --git a/js/models/messages.js b/js/models/messages.js index f67164451..f77af1a50 100644 --- a/js/models/messages.js +++ b/js/models/messages.js @@ -22,7 +22,9 @@ const { deleteExternalMessageFiles, getAbsoluteAttachmentPath, - } = Signal.Migrations; + loadAttachmentData, + loadQuoteData, + } = window.Signal.Migrations; window.AccountCache = Object.create(null); window.AccountJobs = Object.create(null); @@ -54,8 +56,6 @@ window.hasSignalAccount = number => window.AccountCache[number]; window.Whisper.Message = Backbone.Model.extend({ - database: Whisper.Database, - storeName: 'messages', initialize(attributes) { if (_.isObject(attributes)) { this.set( @@ -211,9 +211,11 @@ return ''; }, onDestroy() { + this.cleanup(); + }, + async cleanup() { this.unload(); - - return deleteExternalMessageFiles(this.attributes); + await deleteExternalMessageFiles(this.attributes); }, unload() { if (this.quotedMessage) { @@ -269,16 +271,16 @@ disabled, }; - if (source === this.OUR_NUMBER) { - return { - ...basicProps, - type: 'fromMe', - }; - } else if (fromSync) { + if (fromSync) { return { ...basicProps, type: 'fromSync', }; + } else if (source === this.OUR_NUMBER) { + return { + ...basicProps, + type: 'fromMe', + }; } return basicProps; @@ -416,7 +418,7 @@ const authorColor = contactModel ? contactModel.getColor() : null; const authorAvatar = contactModel ? contactModel.getAvatar() : null; - const authorAvatarPath = authorAvatar.url; + const authorAvatarPath = authorAvatar ? authorAvatar.url : null; const expirationLength = this.get('expireTimer') * 1000; const expireTimerStart = this.get('expirationStartTimestamp'); @@ -654,15 +656,117 @@ contacts: sortedContacts, }; }, - retrySend() { - const retries = _.filter( + + // One caller today: event handler for the 'Retry Send' entry in triple-dot menu + async retrySend() { + const [retries, errors] = _.partition( this.get('errors'), this.isReplayableError.bind(this) ); - _.map(retries, 'number').forEach(number => { - this.resend(number); - }); + + // Remove the errors that aren't replayable + this.set({ errors }); + + const profileKey = null; + const numbers = retries.map(retry => retry.number); + + if (!numbers.length) { + window.log.error( + 'retrySend: Attempted to retry, but no numbers to send to!' + ); + return null; + } + + const attachmentsWithData = await Promise.all( + (this.get('attachments') || []).map(loadAttachmentData) + ); + const quoteWithData = await loadQuoteData(this.get('quote')); + + const conversation = this.getConversation(); + let promise; + + if (conversation.isPrivate()) { + const [number] = numbers; + + promise = textsecure.messaging.sendMessageToNumber( + number, + this.get('body'), + attachmentsWithData, + quoteWithData, + this.get('sent_at'), + this.get('expireTimer'), + profileKey + ); + } else { + // Because this is a partial group send, we manually construct the request like + // sendMessageToGroup does. + promise = textsecure.messaging.sendMessage({ + recipients: numbers, + body: this.get('body'), + timestamp: this.get('sent_at'), + attachments: attachmentsWithData, + quote: quoteWithData, + needsSync: !this.get('synced'), + expireTimer: this.get('expireTimer'), + profileKey, + group: { + id: this.get('conversationId'), + type: textsecure.protobuf.GroupContext.Type.DELIVER, + }, + }); + } + + return this.send(promise); }, + isReplayableError(e) { + return ( + e.name === 'MessageError' || + e.name === 'OutgoingMessageError' || + e.name === 'SendMessageNetworkError' || + e.name === 'SignedPreKeyRotationError' || + e.name === 'OutgoingIdentityKeyError' + ); + }, + + // Called when the user ran into an error with a specific user, wants to send to them + // One caller today: ConversationView.forceSend() + async resend(number) { + const error = this.removeOutgoingErrors(number); + if (error) { + const profileKey = null; + const attachmentsWithData = await Promise.all( + (this.get('attachments') || []).map(loadAttachmentData) + ); + const quoteWithData = await loadQuoteData(this.get('quote')); + + const promise = textsecure.messaging.sendMessageToNumber( + number, + this.get('body'), + attachmentsWithData, + quoteWithData, + this.get('sent_at'), + this.get('expireTimer'), + profileKey + ); + + this.send(promise); + } + }, + removeOutgoingErrors(number) { + const errors = _.partition( + this.get('errors'), + e => + e.number === number && + (e.name === 'MessageError' || + e.name === 'OutgoingMessageError' || + e.name === 'SendMessageNetworkError' || + e.name === 'SignedPreKeyRotationError' || + e.name === 'OutgoingIdentityKeyError') + ); + this.set({ errors: errors[1] }); + return errors[0][0]; + }, + getConversation() { // This needs to be an unsafe call, because this method is called during // initial module setup. We may be in the middle of the initial fetch to @@ -720,9 +824,12 @@ .then(async result => { const now = Date.now(); this.trigger('done'); + + // This is used by sendSyncMessage, then set to null if (result.dataMessage) { this.set({ dataMessage: result.dataMessage }); } + const sentTo = this.get('sent_to') || []; this.set({ sent_to: _.union(sentTo, result.successfulNumbers), @@ -739,6 +846,7 @@ .catch(result => { const now = Date.now(); this.trigger('done'); + if (result.dataMessage) { this.set({ dataMessage: result.dataMessage }); } @@ -774,9 +882,9 @@ ); } - return Promise.all(promises).then(() => { - this.trigger('send-error', this.get('errors')); - }); + this.trigger('send-error', this.get('errors')); + + return Promise.all(promises); }); }, @@ -855,7 +963,6 @@ Message: Whisper.Message, }); }, - hasNetworkError() { const error = _.find( this.get('errors'), @@ -867,36 +974,6 @@ ); return !!error; }, - removeOutgoingErrors(number) { - const errors = _.partition( - this.get('errors'), - e => - e.number === number && - (e.name === 'MessageError' || - e.name === 'OutgoingMessageError' || - e.name === 'SendMessageNetworkError' || - e.name === 'SignedPreKeyRotationError' || - e.name === 'OutgoingIdentityKeyError') - ); - this.set({ errors: errors[1] }); - return errors[0][0]; - }, - isReplayableError(e) { - return ( - e.name === 'MessageError' || - e.name === 'OutgoingMessageError' || - e.name === 'SendMessageNetworkError' || - e.name === 'SignedPreKeyRotationError' || - e.name === 'OutgoingIdentityKeyError' - ); - }, - resend(number) { - const error = this.removeOutgoingErrors(number); - if (error) { - const promise = new textsecure.ReplayableError(error).replay(); - this.send(promise); - } - }, handleDataMessage(dataMessage, confirm) { // This function is called from the background script in a few scenarios: // 1. on an incoming message @@ -1217,10 +1294,12 @@ const expiresAt = start + delta; this.set({ expires_at: expiresAt }); - const id = await window.Signal.Data.saveMessage(this.attributes, { - Message: Whisper.Message, - }); - this.set({ id }); + const id = this.get('id'); + if (id) { + await window.Signal.Data.saveMessage(this.attributes, { + Message: Whisper.Message, + }); + } Whisper.ExpiringMessagesListener.update(); window.log.info('Set message expiration', { @@ -1233,6 +1312,7 @@ Whisper.MessageCollection = Backbone.Collection.extend({ model: Whisper.Message, + // Keeping this for legacy upgrade pre-migrate to SQLCipher database: Whisper.Database, storeName: 'messages', comparator(left, right) { @@ -1282,7 +1362,15 @@ } ); - this.add(messages.models); + const models = messages.filter(message => Boolean(message.id)); + const eliminated = messages.length - models.length; + if (eliminated > 0) { + window.log.warn( + `fetchConversation: Eliminated ${eliminated} messages without an id` + ); + } + + this.add(models); if (unreadCount <= 0) { return; diff --git a/js/modules/backup.js b/js/modules/backup.js index bc44b631e..0c5a46d5f 100644 --- a/js/modules/backup.js +++ b/js/modules/backup.js @@ -13,6 +13,7 @@ const fs = require('fs'); const path = require('path'); +const { map, fromPairs } = require('lodash'); const tmp = require('tmp'); const pify = require('pify'); const archiver = require('archiver'); @@ -1140,12 +1141,13 @@ function getMessageKey(message) { const sourceDevice = message.sourceDevice || 1; return `${source}.${sourceDevice} ${message.timestamp}`; } -function loadMessagesLookup(db) { - return window.Signal.Data.getAllMessageIds({ +async function loadMessagesLookup(db) { + const array = await window.Signal.Data.getAllMessageIds({ db, getMessageKey, handleDOMException: Whisper.Database.handleDOMException, }); + return fromPairs(map(array, item => [item, true])); } function getConversationKey(conversation) { diff --git a/js/modules/data.js b/js/modules/data.js index cdf967ac5..ab15ddddb 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -1,75 +1,233 @@ -/* global window */ +/* global window, setTimeout */ + +const electron = require('electron'); +const { forEach, isFunction, isObject } = require('lodash'); const { deferredToPromise } = require('./deferred_to_promise'); const MessageType = require('./types/message'); -// calls to search for: +const { ipcRenderer } = electron; + +// We listen to a lot of events on ipcRenderer, often on the same channel. This prevents +// any warnings that might be sent to the console in that case. +ipcRenderer.setMaxListeners(0); + +// calls to search for when finding functions to convert: // .fetch( // .save( // .destroy( -async function saveMessage(data, { Message }) { - const message = new Message(data); - await deferredToPromise(message.save()); - return message.id; +const SQL_CHANNEL_KEY = 'sql-channel'; +const ERASE_SQL_KEY = 'erase-sql-key'; +const ERASE_ATTACHMENTS_KEY = 'erase-attachments'; + +const _jobs = Object.create(null); +const _DEBUG = false; +let _jobCounter = 0; + +const channels = {}; + +module.exports = { + _jobs, + _cleanData, + + close, + removeDB, + + saveMessage, + saveMessages, + removeMessage, + getUnreadByConversation, + + removeAllMessagesInConversation, + + getMessageBySender, + getMessageById, + getAllMessageIds, + getMessagesBySentAt, + getExpiredMessages, + getNextExpiringMessage, + getMessagesByConversation, + + getAllUnprocessed, + getUnprocessedById, + saveUnprocessed, + saveUnprocesseds, + updateUnprocessed, + removeUnprocessed, + removeAllUnprocessed, + + removeAll, + removeOtherData, + + // Returning plain JSON + getMessagesNeedingUpgrade, + getLegacyMessagesNeedingUpgrade, + getMessagesWithVisualMediaAttachments, + getMessagesWithFileAttachments, +}; + +// When IPC arguments are prepared for the cross-process send, they are JSON.stringified. +// We can't send ArrayBuffers or BigNumbers (what we get from proto library for dates). +function _cleanData(data) { + const keys = Object.keys(data); + for (let index = 0, max = keys.length; index < max; index += 1) { + const key = keys[index]; + const value = data[key]; + + if (value === null || value === undefined) { + // eslint-disable-next-line no-continue + continue; + } + + if (isFunction(value.toNumber)) { + // eslint-disable-next-line no-param-reassign + data[key] = value.toNumber(); + } else if (Array.isArray(value)) { + // eslint-disable-next-line no-param-reassign + data[key] = value.map(item => _cleanData(item)); + } else if (isObject(value)) { + // eslint-disable-next-line no-param-reassign + data[key] = _cleanData(value); + } else if ( + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' + ) { + window.log.info(`_cleanData: key ${key} had type ${typeof value}`); + } + } + return data; +} + +function _makeJob(fnName) { + _jobCounter += 1; + const id = _jobCounter; + + _jobs[id] = { + fnName, + }; + + return id; +} + +function _updateJob(id, data) { + const { resolve, reject } = data; + + _jobs[id] = { + ..._jobs[id], + ...data, + resolve: value => { + _removeJob(id); + return resolve(value); + }, + reject: error => { + _removeJob(id); + return reject(error); + }, + }; +} + +function _removeJob(id) { + if (_DEBUG) { + _jobs[id].complete = true; + } else { + delete _jobs[id]; + } +} + +function _getJob(id) { + return _jobs[id]; +} + +ipcRenderer.on( + `${SQL_CHANNEL_KEY}-done`, + (event, jobId, errorForDisplay, result) => { + const job = _getJob(jobId); + if (!job) { + throw new Error( + `Received job reply to job ${jobId}, but did not have it in our registry!` + ); + } + + const { resolve, reject, fnName } = job; + + if (errorForDisplay) { + return reject( + new Error(`Error calling channel ${fnName}: ${errorForDisplay}`) + ); + } + + return resolve(result); + } +); + +function makeChannel(fnName) { + channels[fnName] = (...args) => { + const jobId = _makeJob(fnName); + + return new Promise((resolve, reject) => { + ipcRenderer.send(SQL_CHANNEL_KEY, jobId, fnName, ...args); + + _updateJob(jobId, { + resolve, + reject, + args: _DEBUG ? args : null, + }); + + setTimeout( + () => resolve(new Error(`Request to ${fnName} timed out`)), + 5000 + ); + }); + }; +} + +forEach(module.exports, fn => { + if (isFunction(fn)) { + makeChannel(fn.name); + } +}); + +// Note: will need to restart the app after calling this, to set up afresh +async function close() { + await channels.close(); +} + +// Note: will need to restart the app after calling this, to set up afresh +async function removeDB() { + await channels.removeDB(); +} + +async function saveMessage(data, { forceSave } = {}) { + const id = await channels.saveMessage(_cleanData(data), { forceSave }); + return id; +} + +async function saveMessages(arrayOfMessages, { forceSave } = {}) { + await channels.saveMessages(_cleanData(arrayOfMessages), { forceSave }); } async function removeMessage(id, { Message }) { const message = await getMessageById(id, { Message }); + // Note: It's important to have a fully database-hydrated model to delete here because // it needs to delete all associated on-disk files along with the database delete. if (message) { - await deferredToPromise(message.destroy()); + await channels.removeMessage(id); + const model = new Message(message); + await model.cleanup(); } } async function getMessageById(id, { Message }) { - const message = new Message({ id }); - try { - await deferredToPromise(message.fetch()); - return message; - } catch (error) { - return null; - } + const message = await channels.getMessageById(id); + return new Message(message); } -async function getAllMessageIds({ db, handleDOMException, getMessageKey }) { - const lookup = Object.create(null); - const storeName = 'messages'; - - return new Promise((resolve, reject) => { - const transaction = db.transaction(storeName, 'readwrite'); - transaction.onerror = () => { - handleDOMException( - `assembleLookup(${storeName}) transaction error`, - transaction.error, - reject - ); - }; - transaction.oncomplete = () => { - // not really very useful - fires at unexpected times - }; - - const store = transaction.objectStore(storeName); - const request = store.openCursor(); - request.onerror = () => { - handleDOMException( - `assembleLookup(${storeName}) request error`, - request.error, - reject - ); - }; - request.onsuccess = event => { - const cursor = event.target.result; - if (cursor && cursor.value) { - lookup[getMessageKey(cursor.value)] = true; - cursor.continue(); - } else { - window.log.info(`Done creating ${storeName} lookup`); - resolve(lookup); - } - }; - }); +async function getAllMessageIds() { + const ids = await channels.getAllMessageIds(); + return ids; } async function getMessageBySender( @@ -77,186 +235,155 @@ async function getMessageBySender( { source, sourceDevice, sent_at }, { Message } ) { - const fetcher = new Message(); - const options = { - index: { - name: 'unique', - // eslint-disable-next-line camelcase - value: [source, sourceDevice, sent_at], - }, - }; - - try { - await deferredToPromise(fetcher.fetch(options)); - if (fetcher.get('id')) { - return fetcher; - } - - return null; - } catch (error) { + const messages = await channels.getMessageBySender({ + source, + sourceDevice, + sent_at, + }); + if (!messages || !messages.length) { return null; } + + return new Message(messages[0]); } async function getUnreadByConversation(conversationId, { MessageCollection }) { - const messages = new MessageCollection(); - - await deferredToPromise( - messages.fetch({ - index: { - // 'unread' index - name: 'unread', - lower: [conversationId], - upper: [conversationId, Number.MAX_VALUE], - }, - }) - ); - - return messages; + const messages = await channels.getUnreadByConversation(conversationId); + return new MessageCollection(messages); } async function getMessagesByConversation( conversationId, { limit = 100, receivedAt = Number.MAX_VALUE, MessageCollection } ) { - const messages = new MessageCollection(); - - const options = { + const messages = await channels.getMessagesByConversation(conversationId, { limit, - index: { - // 'conversation' index on [conversationId, received_at] - name: 'conversation', - lower: [conversationId], - upper: [conversationId, receivedAt], - order: 'desc', - // SELECT messages WHERE conversationId = this.id ORDER - // received_at DESC - }, - }; - await deferredToPromise(messages.fetch(options)); + receivedAt, + }); - return messages; + return new MessageCollection(messages); } async function removeAllMessagesInConversation( conversationId, { MessageCollection } ) { - const messages = new MessageCollection(); - - let loaded; + let messages; do { // Yes, we really want the await in the loop. We're deleting 100 at a // time so we don't use too much memory. // eslint-disable-next-line no-await-in-loop - await deferredToPromise( - messages.fetch({ - limit: 100, - index: { - // 'conversation' index on [conversationId, received_at] - name: 'conversation', - lower: [conversationId], - upper: [conversationId, Number.MAX_VALUE], - }, - }) - ); + messages = await getMessagesByConversation(conversationId, { + limit: 100, + MessageCollection, + }); - loaded = messages.models; - messages.reset([]); + if (!messages.length) { + return; + } + + const ids = messages.map(message => message.id); // Note: It's very important that these models are fully hydrated because // we need to delete all associated on-disk files along with the database delete. - loaded.map(message => message.destroy()); - } while (loaded.length > 0); + // eslint-disable-next-line no-await-in-loop + await Promise.all(messages.map(message => message.cleanup())); + + // eslint-disable-next-line no-await-in-loop + await channels.removeMessage(ids); + } while (messages.length > 0); } async function getMessagesBySentAt(sentAt, { MessageCollection }) { - const messages = new MessageCollection(); - - await deferredToPromise( - messages.fetch({ - index: { - // 'receipt' index on sent_at - name: 'receipt', - only: sentAt, - }, - }) - ); - - return messages; + const messages = await channels.getMessagesBySentAt(sentAt); + return new MessageCollection(messages); } async function getExpiredMessages({ MessageCollection }) { window.log.info('Load expired messages'); - const messages = new MessageCollection(); - - await deferredToPromise( - messages.fetch({ - conditions: { - expires_at: { - $lte: Date.now(), - }, - }, - }) - ); - - return messages; + const messages = await channels.getExpiredMessages(); + return new MessageCollection(messages); } async function getNextExpiringMessage({ MessageCollection }) { - const messages = new MessageCollection(); - - await deferredToPromise( - messages.fetch({ - limit: 1, - index: { - name: 'expires_at', - }, - }) - ); - - return messages; + const messages = await channels.getNextExpiringMessage(); + return new MessageCollection(messages); } -async function saveUnprocessed(data, { Unprocessed }) { - const unprocessed = new Unprocessed(data); - return deferredToPromise(unprocessed.save()); +async function getAllUnprocessed() { + return channels.getAllUnprocessed(); } -async function getAllUnprocessed({ UnprocessedCollection }) { - const collection = new UnprocessedCollection(); - await deferredToPromise(collection.fetch()); - return collection.map(model => model.attributes); +async function getUnprocessedById(id, { Unprocessed }) { + const unprocessed = await channels.getUnprocessedById(id); + return new Unprocessed(unprocessed); } -async function updateUnprocessed(id, updates, { Unprocessed }) { - const unprocessed = new Unprocessed({ - id, +async function saveUnprocessed(data, { forceSave } = {}) { + const id = await channels.saveUnprocessed(_cleanData(data), { forceSave }); + return id; +} + +async function saveUnprocesseds(arrayOfUnprocessed, { forceSave } = {}) { + await channels.saveUnprocesseds(_cleanData(arrayOfUnprocessed), { + forceSave, }); - - await deferredToPromise(unprocessed.fetch()); - - unprocessed.set(updates); - await saveUnprocessed(unprocessed.attributes, { Unprocessed }); } -async function removeUnprocessed(id, { Unprocessed }) { - const unprocessed = new Unprocessed({ - id, - }); +async function updateUnprocessed(id, updates) { + const existing = await channels.getUnprocessedById(id); + if (!existing) { + throw new Error(`Unprocessed id ${id} does not exist in the database!`); + } + const toSave = { + ...existing, + ...updates, + }; - await deferredToPromise(unprocessed.destroy()); + await saveUnprocessed(toSave); +} + +async function removeUnprocessed(id) { + await channels.removeUnprocessed(id); } async function removeAllUnprocessed() { - // erase everything in unprocessed table + await channels.removeAllUnprocessed(); } async function removeAll() { - // erase everything in the database + await channels.removeAll(); } -async function getMessagesNeedingUpgrade( +// Note: will need to restart the app after calling this, to set up afresh +async function removeOtherData() { + await Promise.all([ + callChannel(ERASE_SQL_KEY), + callChannel(ERASE_ATTACHMENTS_KEY), + ]); +} + +async function callChannel(name) { + return new Promise((resolve, reject) => { + ipcRenderer.send(name); + ipcRenderer.once(`${name}-done`, (event, error) => { + if (error) { + return reject(error); + } + + return resolve(); + }); + + setTimeout( + () => reject(new Error(`callChannel call to ${name} timed out`)), + 5000 + ); + }); +} + +// Functions below here return JSON + +async function getLegacyMessagesNeedingUpgrade( limit, { MessageCollection, maxVersion = MessageType.CURRENT_SCHEMA_VERSION } ) { @@ -278,75 +405,28 @@ async function getMessagesNeedingUpgrade( return models.map(model => model.toJSON()); } +async function getMessagesNeedingUpgrade( + limit, + { maxVersion = MessageType.CURRENT_SCHEMA_VERSION } +) { + const messages = await channels.getMessagesNeedingUpgrade(limit, { + maxVersion, + }); + + return messages; +} + async function getMessagesWithVisualMediaAttachments( conversationId, - { limit, MessageCollection } + { limit } ) { - const messages = new MessageCollection(); - const lowerReceivedAt = 0; - const upperReceivedAt = Number.MAX_VALUE; - - await deferredToPromise( - messages.fetch({ - limit, - index: { - name: 'hasVisualMediaAttachments', - lower: [conversationId, lowerReceivedAt, 1], - upper: [conversationId, upperReceivedAt, 1], - order: 'desc', - }, - }) - ); - - return messages.models.map(model => model.toJSON()); + return channels.getMessagesWithVisualMediaAttachments(conversationId, { + limit, + }); } -async function getMessagesWithFileAttachments( - conversationId, - { limit, MessageCollection } -) { - const messages = new MessageCollection(); - const lowerReceivedAt = 0; - const upperReceivedAt = Number.MAX_VALUE; - - await deferredToPromise( - messages.fetch({ - limit, - index: { - name: 'hasFileAttachments', - lower: [conversationId, lowerReceivedAt, 1], - upper: [conversationId, upperReceivedAt, 1], - order: 'desc', - }, - }) - ); - - return messages.models.map(model => model.toJSON()); +async function getMessagesWithFileAttachments(conversationId, { limit }) { + return channels.getMessagesWithFileAttachments(conversationId, { + limit, + }); } - -module.exports = { - saveMessage, - removeMessage, - getUnreadByConversation, - removeAllMessagesInConversation, - getMessageBySender, - getMessageById, - getAllMessageIds, - getMessagesBySentAt, - getExpiredMessages, - getNextExpiringMessage, - getMessagesByConversation, - - getAllUnprocessed, - saveUnprocessed, - updateUnprocessed, - removeUnprocessed, - removeAllUnprocessed, - - removeAll, - - // Returning plain JSON - getMessagesNeedingUpgrade, - getMessagesWithVisualMediaAttachments, - getMessagesWithFileAttachments, -}; diff --git a/js/modules/migrate_to_sql.js b/js/modules/migrate_to_sql.js new file mode 100644 index 000000000..f2ceb3590 --- /dev/null +++ b/js/modules/migrate_to_sql.js @@ -0,0 +1,167 @@ +/* global window, IDBKeyRange */ + +const { includes, isFunction, isString, last } = require('lodash'); +const { saveMessages, saveUnprocesseds } = require('./data'); +const { + getMessageExportLastIndex, + setMessageExportLastIndex, + getUnprocessedExportLastIndex, + setUnprocessedExportLastIndex, +} = require('./settings'); + +module.exports = { + migrateToSQL, +}; + +async function migrateToSQL({ db, clearStores, handleDOMException }) { + if (!db) { + throw new Error('Need db for IndexedDB connection!'); + } + if (!isFunction(clearStores)) { + throw new Error('Need clearStores function!'); + } + if (!isFunction(handleDOMException)) { + throw new Error('Need handleDOMException function!'); + } + + window.log.info('migrateToSQL: start'); + + let lastIndex = await getMessageExportLastIndex(db); + let complete = false; + + while (!complete) { + // eslint-disable-next-line no-await-in-loop + const status = await migrateStoreToSQLite({ + db, + save: saveMessages, + storeName: 'messages', + handleDOMException, + lastIndex, + }); + + ({ complete, lastIndex } = status); + + // eslint-disable-next-line no-await-in-loop + await setMessageExportLastIndex(db, lastIndex); + } + window.log.info('migrateToSQL: migrate of messages complete'); + + lastIndex = await getUnprocessedExportLastIndex(db); + complete = false; + + while (!complete) { + // eslint-disable-next-line no-await-in-loop + const status = await migrateStoreToSQLite({ + db, + save: saveUnprocesseds, + storeName: 'unprocessed', + handleDOMException, + lastIndex, + }); + + ({ complete, lastIndex } = status); + + // eslint-disable-next-line no-await-in-loop + await setUnprocessedExportLastIndex(db, lastIndex); + } + window.log.info('migrateToSQL: migrate of unprocessed complete'); + + await clearStores(['messages', 'unprocessed']); + + window.log.info('migrateToSQL: complete'); +} + +async function migrateStoreToSQLite({ + db, + save, + storeName, + handleDOMException, + lastIndex = null, + batchSize = 20, +}) { + if (!db) { + throw new Error('Need db for IndexedDB connection!'); + } + if (!isFunction(save)) { + throw new Error('Need save function!'); + } + if (!isString(storeName)) { + throw new Error('Need storeName!'); + } + if (!isFunction(handleDOMException)) { + throw new Error('Need handleDOMException for error handling!'); + } + + if (!includes(db.objectStoreNames, storeName)) { + return { + complete: true, + count: 0, + }; + } + + const queryPromise = new Promise((resolve, reject) => { + const items = []; + const transaction = db.transaction(storeName, 'readonly'); + transaction.onerror = () => { + handleDOMException( + 'migrateToSQLite transaction error', + transaction.error, + reject + ); + }; + transaction.oncomplete = () => {}; + + const store = transaction.objectStore(storeName); + const excludeLowerBound = true; + const range = lastIndex + ? IDBKeyRange.lowerBound(lastIndex, excludeLowerBound) + : undefined; + const request = store.openCursor(range); + request.onerror = () => { + handleDOMException( + 'migrateToSQLite: request error', + request.error, + reject + ); + }; + request.onsuccess = event => { + const cursor = event.target.result; + + if (!cursor || !cursor.value) { + return resolve({ + complete: true, + items, + }); + } + + const item = cursor.value; + items.push(item); + + if (items.length >= batchSize) { + return resolve({ + complete: false, + items, + }); + } + + return cursor.continue(); + }; + }); + + const { items, complete } = await queryPromise; + + if (items.length) { + // We need to pass forceSave parameter, because these items already have an + // id key. Normally, this call would be interpreted as an update request. + await save(items, { forceSave: true }); + } + + const lastItem = last(items); + const id = lastItem ? lastItem.id : null; + + return { + complete, + count: items.length, + lastIndex: id, + }; +} diff --git a/js/modules/settings.js b/js/modules/settings.js index 334123877..2fa2310a7 100644 --- a/js/modules/settings.js +++ b/js/modules/settings.js @@ -3,25 +3,34 @@ const { isObject, isString } = require('lodash'); const ITEMS_STORE_NAME = 'items'; const LAST_PROCESSED_INDEX_KEY = 'attachmentMigration_lastProcessedIndex'; const IS_MIGRATION_COMPLETE_KEY = 'attachmentMigration_isComplete'; +const MESSAGE_LAST_INDEX_KEY = 'sqlMigration_messageLastIndex'; +const UNPROCESSED_LAST_INDEX_KEY = 'sqlMigration_unprocessedLastIndex'; // Public API exports.READ_RECEIPT_CONFIGURATION_SYNC = 'read-receipt-configuration-sync'; exports.getAttachmentMigrationLastProcessedIndex = connection => exports._getItem(connection, LAST_PROCESSED_INDEX_KEY); - exports.setAttachmentMigrationLastProcessedIndex = (connection, value) => exports._setItem(connection, LAST_PROCESSED_INDEX_KEY, value); - exports.deleteAttachmentMigrationLastProcessedIndex = connection => exports._deleteItem(connection, LAST_PROCESSED_INDEX_KEY); exports.isAttachmentMigrationComplete = async connection => Boolean(await exports._getItem(connection, IS_MIGRATION_COMPLETE_KEY)); - exports.markAttachmentMigrationComplete = connection => exports._setItem(connection, IS_MIGRATION_COMPLETE_KEY, true); +exports.getMessageExportLastIndex = connection => + exports._getItem(connection, MESSAGE_LAST_INDEX_KEY); +exports.setMessageExportLastIndex = (connection, lastIndex) => + exports._setItem(connection, MESSAGE_LAST_INDEX_KEY, lastIndex); + +exports.getUnprocessedExportLastIndex = connection => + exports._getItem(connection, UNPROCESSED_LAST_INDEX_KEY); +exports.setUnprocessedExportLastIndex = (connection, lastIndex) => + exports._setItem(connection, UNPROCESSED_LAST_INDEX_KEY, lastIndex); + // Private API exports._getItem = (connection, key) => { if (!isObject(connection)) { diff --git a/js/modules/signal.js b/js/modules/signal.js index bfd61b736..c574301ea 100644 --- a/js/modules/signal.js +++ b/js/modules/signal.js @@ -10,6 +10,7 @@ const OS = require('../../ts/OS'); const Settings = require('./settings'); const Startup = require('./startup'); const Util = require('../../ts/util'); +const { migrateToSQL } = require('./migrate_to_sql'); // Components const { @@ -110,6 +111,7 @@ function initializeMigrations({ const attachmentsPath = getPath(userDataPath); const readAttachmentData = createReader(attachmentsPath); const loadAttachmentData = Type.loadData(readAttachmentData); + const loadQuoteData = MessageType.loadQuoteData(readAttachmentData); const getAbsoluteAttachmentPath = createAbsolutePathGetter(attachmentsPath); const deleteOnDisk = Attachments.createDeleter(attachmentsPath); @@ -122,6 +124,7 @@ function initializeMigrations({ getAbsoluteAttachmentPath, getPlaceholderMigrations, loadAttachmentData, + loadQuoteData, loadMessage: MessageType.createAttachmentLoader(loadAttachmentData), Migrations0DatabaseWithAttachmentData, Migrations1DatabaseWithoutAttachmentData, @@ -222,5 +225,6 @@ exports.setup = (options = {}) => { Util, Views, Workflow, + migrateToSQL, }; }; diff --git a/js/modules/types/message.js b/js/modules/types/message.js index cd0609083..b757e1708 100644 --- a/js/modules/types/message.js +++ b/js/modules/types/message.js @@ -166,7 +166,7 @@ exports._mapAttachments = upgradeAttachment => async (message, context) => { const upgradeWithContext = attachment => upgradeAttachment(attachment, context); const attachments = await Promise.all( - message.attachments.map(upgradeWithContext) + (message.attachments || []).map(upgradeWithContext) ); return Object.assign({}, message, { attachments }); }; @@ -356,7 +356,9 @@ exports.upgradeSchema = async ( exports.createAttachmentLoader = loadAttachmentData => { if (!isFunction(loadAttachmentData)) { - throw new TypeError('`loadAttachmentData` is required'); + throw new TypeError( + 'createAttachmentLoader: loadAttachmentData is required' + ); } return async message => @@ -367,6 +369,36 @@ exports.createAttachmentLoader = loadAttachmentData => { }); }; +exports.loadQuoteData = loadAttachmentData => { + if (!isFunction(loadAttachmentData)) { + throw new TypeError('loadQuoteData: loadAttachmentData is required'); + } + + return async quote => { + if (!quote) { + return null; + } + + return { + ...quote, + attachments: await Promise.all( + (quote.attachments || []).map(async attachment => { + const { thumbnail } = attachment; + + if (!thumbnail || !thumbnail.path) { + return attachment; + } + + return { + ...attachment, + thumbnail: await loadAttachmentData(thumbnail), + }; + }) + ), + }; + }; +}; + exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => { if (!isFunction(deleteAttachmentData)) { throw new TypeError( @@ -392,7 +424,7 @@ exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => { quote.attachments.map(async attachment => { const { thumbnail } = attachment; - if (thumbnail.path) { + if (thumbnail && thumbnail.path) { await deleteOnDisk(thumbnail.path); } }) diff --git a/js/signal_protocol_store.js b/js/signal_protocol_store.js index e8a81afc4..12ba9b1a7 100644 --- a/js/signal_protocol_store.js +++ b/js/signal_protocol_store.js @@ -127,13 +127,7 @@ return this.fetch({ range: [`${number}.1`, `${number}.:`] }); }, }); - const Unprocessed = Model.extend({ storeName: 'unprocessed' }); - const UnprocessedCollection = Backbone.Collection.extend({ - storeName: 'unprocessed', - database: Whisper.Database, - model: Unprocessed, - comparator: 'timestamp', - }); + const Unprocessed = Model.extend(); const IdentityRecord = Model.extend({ storeName: 'identityKeys', validAttributes: [ @@ -946,10 +940,15 @@ // Not yet processed messages - for resiliency getAllUnprocessed() { - return window.Signal.Data.getAllUnprocessed({ UnprocessedCollection }); + return window.Signal.Data.getAllUnprocessed(); }, addUnprocessed(data) { - return window.Signal.Data.saveUnprocessed(data, { Unprocessed }); + // We need to pass forceSave because the data has an id already, which will cause + // an update instead of an insert. + return window.Signal.Data.saveUnprocessed(data, { + forceSave: true, + Unprocessed, + }); }, updateUnprocessed(id, updates) { return window.Signal.Data.updateUnprocessed(id, updates, { Unprocessed }); @@ -961,6 +960,7 @@ // First the in-memory caches: window.storage.reset(); // items store ConversationController.reset(); // conversations store + await ConversationController.load(); // Then, the entire database: await Whisper.Database.clear(); diff --git a/js/views/clear_data_view.js b/js/views/clear_data_view.js index ec132c9e5..e7b30cd26 100644 --- a/js/views/clear_data_view.js +++ b/js/views/clear_data_view.js @@ -46,7 +46,15 @@ }, async clearAllData() { try { - await Promise.all([Logs.deleteAll(), Database.drop()]); + await Promise.all([ + Logs.deleteAll(), + Database.drop(), + window.Signal.Data.removeAll(), + window.Signal.Data.removeOtherData(), + ]); + + await window.Signal.Data.close(); + await window.Signal.Data.removeDB(); } catch (error) { window.log.error( 'Something went wrong deleting all data:', diff --git a/js/views/conversation_view.js b/js/views/conversation_view.js index 799c7086d..5c3792d5d 100644 --- a/js/views/conversation_view.js +++ b/js/views/conversation_view.js @@ -515,9 +515,7 @@ const messagesLoaded = this.inProgressFetch || Promise.resolve(); // eslint-disable-next-line more/no-then - messagesLoaded - .then(this.model.decryptOldIncomingKeyErrors.bind(this)) - .then(this.onLoaded.bind(this), this.onLoaded.bind(this)); + messagesLoaded.then(this.onLoaded.bind(this), this.onLoaded.bind(this)); this.view.resetScrollPosition(); this.$el.trigger('force-resize'); @@ -799,11 +797,16 @@ this.inProgressFetch = this.model .fetchContacts() .then(() => this.model.fetchMessages()) - .then(() => { + .then(async () => { this.$('.bar-container').hide(); - this.model.messageCollection.where({ unread: 1 }).forEach(m => { - m.fetch(); - }); + await Promise.all( + this.model.messageCollection.where({ unread: 1 }).map(async m => { + const latest = await window.Signal.Data.getMessageById(m.id, { + Message: Whisper.Message, + }); + m.merge(latest); + }) + ); this.inProgressFetch = null; }) .catch(error => { @@ -1003,6 +1006,7 @@ Message: Whisper.Message, }); message.trigger('unload'); + this.model.messageCollection.remove(message.id); this.resetPanel(); this.updateHeader(); }, @@ -1138,10 +1142,17 @@ async destroyMessages() { try { await this.confirm(i18n('deleteConversationConfirmation')); - await this.model.destroyMessages(); - this.remove(); + try { + await this.model.destroyMessages(); + this.remove(); + } catch (error) { + window.log.error( + 'destroyMessages: Failed to successfully delete conversation', + error && error.stack ? error.stack : error + ); + } } catch (error) { - // nothing to see here + // nothing to see here, user canceled out of dialog } }, diff --git a/libtextsecure/errors.js b/libtextsecure/errors.js index 7e6221277..66cf29392 100644 --- a/libtextsecure/errors.js +++ b/libtextsecure/errors.js @@ -2,21 +2,7 @@ // eslint-disable-next-line func-names (function() { - const registeredFunctions = {}; - const Type = { - ENCRYPT_MESSAGE: 1, - INIT_SESSION: 2, - TRANSMIT_MESSAGE: 3, - REBUILD_MESSAGE: 4, - RETRY_SEND_MESSAGE_PROTO: 5, - }; window.textsecure = window.textsecure || {}; - window.textsecure.replay = { - Type, - registerFunction(func, functionCode) { - registeredFunctions[functionCode] = func; - }, - }; function inherit(Parent, Child) { // eslint-disable-next-line no-param-reassign @@ -46,23 +32,15 @@ } this.functionCode = options.functionCode; - this.args = options.args; } inherit(Error, ReplayableError); - ReplayableError.prototype.replay = function replay(...argumentsAsArray) { - const args = this.args.concat(argumentsAsArray); - return registeredFunctions[this.functionCode].apply(window, args); - }; - function IncomingIdentityKeyError(number, message, key) { // eslint-disable-next-line prefer-destructuring this.number = number.split('.')[0]; this.identityKey = key; ReplayableError.call(this, { - functionCode: Type.INIT_SESSION, - args: [number, message], name: 'IncomingIdentityKeyError', message: `The identity of ${this.number} has changed.`, }); @@ -75,8 +53,6 @@ this.identityKey = identityKey; ReplayableError.call(this, { - functionCode: Type.ENCRYPT_MESSAGE, - args: [number, message, timestamp], name: 'OutgoingIdentityKeyError', message: `The identity of ${this.number} has changed.`, }); @@ -84,9 +60,10 @@ inherit(ReplayableError, OutgoingIdentityKeyError); function OutgoingMessageError(number, message, timestamp, httpError) { + // eslint-disable-next-line prefer-destructuring + this.number = number.split('.')[0]; + ReplayableError.call(this, { - functionCode: Type.ENCRYPT_MESSAGE, - args: [number, message, timestamp], name: 'OutgoingMessageError', message: httpError ? httpError.message : 'no http error', }); @@ -98,13 +75,11 @@ } inherit(ReplayableError, OutgoingMessageError); - function SendMessageNetworkError(number, jsonData, httpError, timestamp) { + function SendMessageNetworkError(number, jsonData, httpError) { this.number = number; this.code = httpError.code; ReplayableError.call(this, { - functionCode: Type.TRANSMIT_MESSAGE, - args: [number, jsonData, timestamp], name: 'SendMessageNetworkError', message: httpError.message, }); @@ -113,10 +88,8 @@ } inherit(ReplayableError, SendMessageNetworkError); - function SignedPreKeyRotationError(numbers, message, timestamp) { + function SignedPreKeyRotationError() { ReplayableError.call(this, { - functionCode: Type.RETRY_SEND_MESSAGE_PROTO, - args: [numbers, message, timestamp], name: 'SignedPreKeyRotationError', message: 'Too many signed prekey rotation failures', }); @@ -127,8 +100,6 @@ this.code = httpError.code; ReplayableError.call(this, { - functionCode: Type.REBUILD_MESSAGE, - args: [message], name: 'MessageError', message: httpError.message, }); diff --git a/libtextsecure/message_receiver.js b/libtextsecure/message_receiver.js index 830c9ce8d..34e2b20a9 100644 --- a/libtextsecure/message_receiver.js +++ b/libtextsecure/message_receiver.js @@ -292,7 +292,10 @@ MessageReceiver.prototype.extend({ }, stringToArrayBuffer(string) { // eslint-disable-next-line new-cap - return new dcodeIO.ByteBuffer.wrap(string, 'binary').toArrayBuffer(); + return dcodeIO.ByteBuffer.wrap(string, 'binary').toArrayBuffer(); + }, + arrayBufferToString(arrayBuffer) { + return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('binary'); }, getAllFromCache() { window.log.info('getAllFromCache'); @@ -331,7 +334,7 @@ MessageReceiver.prototype.extend({ const id = this.getEnvelopeId(envelope); const data = { id, - envelope: plaintext, + envelope: this.arrayBufferToString(plaintext), timestamp: Date.now(), attempts: 1, }; @@ -340,7 +343,7 @@ MessageReceiver.prototype.extend({ updateCache(envelope, plaintext) { const id = this.getEnvelopeId(envelope); const data = { - decrypted: plaintext, + decrypted: this.arrayBufferToString(plaintext), }; return textsecure.storage.unprocessed.update(id, data); }, @@ -1153,11 +1156,6 @@ textsecure.MessageReceiver = function MessageReceiverWrapper( this.getStatus = messageReceiver.getStatus.bind(messageReceiver); this.close = messageReceiver.close.bind(messageReceiver); messageReceiver.connect(); - - textsecure.replay.registerFunction( - messageReceiver.tryMessageAgain.bind(messageReceiver), - textsecure.replay.Type.INIT_SESSION - ); }; textsecure.MessageReceiver.prototype = { diff --git a/libtextsecure/sendmessage.js b/libtextsecure/sendmessage.js index 13ff05110..72e61ad82 100644 --- a/libtextsecure/sendmessage.js +++ b/libtextsecure/sendmessage.js @@ -705,8 +705,9 @@ MessageSender.prototype = { profileKey ) { return textsecure.storage.groups.getNumbers(groupId).then(targetNumbers => { - if (targetNumbers === undefined) + if (targetNumbers === undefined) { return Promise.reject(new Error('Unknown Group')); + } const me = textsecure.storage.user.getNumber(); const numbers = targetNumbers.filter(number => number !== me); @@ -895,22 +896,6 @@ textsecure.MessageSender = function MessageSenderWrapper( cdnUrl ) { const sender = new MessageSender(url, username, password, cdnUrl); - textsecure.replay.registerFunction( - sender.tryMessageAgain.bind(sender), - textsecure.replay.Type.ENCRYPT_MESSAGE - ); - textsecure.replay.registerFunction( - sender.retransmitMessage.bind(sender), - textsecure.replay.Type.TRANSMIT_MESSAGE - ); - textsecure.replay.registerFunction( - sender.sendMessage.bind(sender), - textsecure.replay.Type.REBUILD_MESSAGE - ); - textsecure.replay.registerFunction( - sender.retrySendMessageProto.bind(sender), - textsecure.replay.Type.RETRY_SEND_MESSAGE_PROTO - ); this.sendExpirationTimerUpdateToNumber = sender.sendExpirationTimerUpdateToNumber.bind( sender diff --git a/main.js b/main.js index 684a814fe..4a80ccd84 100644 --- a/main.js +++ b/main.js @@ -4,6 +4,7 @@ const path = require('path'); const url = require('url'); const os = require('os'); const fs = require('fs'); +const crypto = require('crypto'); const _ = require('lodash'); const pify = require('pify'); @@ -23,7 +24,9 @@ const { const packageJson = require('./package.json'); -const Attachments = require('./app/attachments'); +const sql = require('./app/sql'); +const sqlChannels = require('./app/sql_channel'); +const attachmentChannel = require('./app/attachment_channel'); const autoUpdate = require('./app/auto_update'); const createTrayIcon = require('./app/tray_icon'); const GlobalErrors = require('./app/global_errors'); @@ -596,44 +599,48 @@ app.on('ready', async () => { installPermissionsHandler({ session, userConfig }); - // NOTE: Temporarily allow `then` until we convert the entire file to `async` / `await`: - /* eslint-disable more/no-then */ let loggingSetupError; - logging - .initialize() - .catch(error => { - loggingSetupError = error; - }) - .then(async () => { - /* eslint-enable more/no-then */ - logger = logging.getLogger(); - logger.info('app ready'); + try { + await logging.initialize(); + } catch (error) { + loggingSetupError = error; + } - if (loggingSetupError) { - logger.error('Problem setting up logging', loggingSetupError.stack); - } + logger = logging.getLogger(); + logger.info('app ready'); - if (!locale) { - const appLocale = - process.env.NODE_ENV === 'test' ? 'en' : app.getLocale(); - locale = loadLocale({ appLocale, logger }); - } + if (loggingSetupError) { + logger.error('Problem setting up logging', loggingSetupError.stack); + } - console.log('Ensure attachments directory exists'); - await Attachments.ensureDirectory(userDataPath); + if (!locale) { + const appLocale = process.env.NODE_ENV === 'test' ? 'en' : app.getLocale(); + locale = loadLocale({ appLocale, logger }); + } - ready = true; + await attachmentChannel.initialize({ configDir: userDataPath }); - autoUpdate.initialize(getMainWindow, locale.messages); + let key = userConfig.get('key'); + if (!key) { + // https://www.zetetic.net/sqlcipher/sqlcipher-api/#key + key = crypto.randomBytes(32).toString('hex'); + userConfig.set('key', key); + } - createWindow(); + await sql.initialize({ configDir: userDataPath, key }); + await sqlChannels.initialize({ userConfig }); - if (usingTrayIcon) { - tray = createTrayIcon(getMainWindow, locale.messages); - } + ready = true; - setupMenu(); - }); + autoUpdate.initialize(getMainWindow, locale.messages); + + createWindow(); + + if (usingTrayIcon) { + tray = createTrayIcon(getMainWindow, locale.messages); + } + + setupMenu(); }); function setupMenu(options) { diff --git a/package.json b/package.json index c636fc03f..d2d5b9d85 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "electron-config": "^1.0.0", "electron-editor-context-menu": "^1.1.1", "electron-is-dev": "^0.3.0", + "@journeyapps/sqlcipher": "https://github.com/scottnonnenberg-signal/node-sqlcipher.git#9682a5c75b1a52c847cfb4b431eb2163d0eeec93", "electron-unhandled": "https://github.com/scottnonnenberg-signal/electron-unhandled.git#7496187472aa561d39fcd4c843a54ffbef0a388c", "electron-updater": "^2.21.10", "emoji-datasource": "4.0.0", @@ -87,6 +88,7 @@ "tmp": "^0.0.33", "to-arraybuffer": "^1.0.1", "underscore": "^1.9.0", + "uuid": "^3.3.2", "websocket": "^1.0.25" }, "devDependencies": { diff --git a/stylesheets/_modules.scss b/stylesheets/_modules.scss index 99ef36a98..605be3ad6 100644 --- a/stylesheets/_modules.scss +++ b/stylesheets/_modules.scss @@ -2107,6 +2107,9 @@ @include color-svg('../images/read.svg', $color-light-35); width: 18px; } +.module-conversation-list-item__message__status-icon--error { + @include color-svg('../images/error.svg', $color-core-red); +} // Third-party module: react-contextmenu diff --git a/test/_test.js b/test/_test.js index b3315a797..6b97b72c5 100644 --- a/test/_test.js +++ b/test/_test.js @@ -71,6 +71,7 @@ function deleteDatabase() { /* Delete the database before running any tests */ before(async () => { await deleteDatabase(); + await window.Signal.Data.removeAll(); await Signal.Migrations.Migrations0DatabaseWithAttachmentData.run({ Backbone, @@ -82,4 +83,5 @@ before(async () => { async function clearDatabase() { const db = await Whisper.Database.open(); await Whisper.Database.clear(); + await window.Signal.Data.removeAll(); } diff --git a/test/keychange_listener_test.js b/test/keychange_listener_test.js index 1d7e24dab..db208d620 100644 --- a/test/keychange_listener_test.js +++ b/test/keychange_listener_test.js @@ -34,12 +34,11 @@ describe('KeyChangeListener', function() { }); it('generates a key change notice in the private conversation with this contact', function(done) { - convo.on('newmessage', function() { - return convo.fetchMessages().then(function() { - var message = convo.messageCollection.at(0); - assert.strictEqual(message.get('type'), 'keychange'); - done(); - }); + convo.on('newmessage', async function() { + await convo.fetchMessages(); + var message = convo.messageCollection.at(0); + assert.strictEqual(message.get('type'), 'keychange'); + done(); }); store.saveIdentity(address.toString(), newKey); }); @@ -61,12 +60,11 @@ describe('KeyChangeListener', function() { }); it('generates a key change notice in the group conversation with this contact', function(done) { - convo.on('newmessage', function() { - return convo.fetchMessages().then(function() { - var message = convo.messageCollection.at(0); - assert.strictEqual(message.get('type'), 'keychange'); - done(); - }); + convo.on('newmessage', async function() { + await convo.fetchMessages(); + var message = convo.messageCollection.at(0); + assert.strictEqual(message.get('type'), 'keychange'); + done(); }); store.saveIdentity(address.toString(), newKey); }); diff --git a/ts/components/ConversationListItem.md b/ts/components/ConversationListItem.md index e67dd33fe..59d0213b4 100644 --- a/ts/components/ConversationListItem.md +++ b/ts/components/ConversationListItem.md @@ -32,6 +32,73 @@ /> ``` +#### All types of status + +```jsx +
+ console.log('onClick')} + i18n={util.i18n} + /> + console.log('onClick')} + i18n={util.i18n} + /> + console.log('onClick')} + i18n={util.i18n} + /> + console.log('onClick')} + i18n={util.i18n} + /> + console.log('onClick')} + i18n={util.i18n} + /> +
+``` + #### With unread ```jsx @@ -278,5 +345,15 @@ On platforms that show scrollbars all the time, this is true all the time. onClick={() => console.log('onClick')} i18n={util.i18n} /> + console.log('onClick')} + i18n={util.i18n} + /> ``` diff --git a/ts/components/ConversationListItem.tsx b/ts/components/ConversationListItem.tsx index 7b18b1f32..f9d9acae1 100644 --- a/ts/components/ConversationListItem.tsx +++ b/ts/components/ConversationListItem.tsx @@ -139,23 +139,21 @@ export class ConversationListItem extends React.Component { return (
- {lastMessage.text ? ( -
0 - ? 'module-conversation-list-item__message__text--has-unread' - : null - )} - > - -
- ) : null} +
0 + ? 'module-conversation-list-item__message__text--has-unread' + : null + )} + > + +
{lastMessage.status ? (