Migrate to SQLCipher for messages/cache
Quite a few other fixes, including: - Sending to contact with no avatar yet (not synced from mobile) - Left pane doesn't update quickly or at all on new message - Left pane doesn't show sent or error status Also: - Contributing.md: Ensure set of linux dev dependencies is complete
This commit is contained in:
Scott Nonnenberg
parent
fc461c82ce
commit
3105b77475
29 changed files with 2006 additions and 716 deletions
|
|
@ -13,6 +13,7 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { map, fromPairs } = require('lodash');
|
||||
const tmp = require('tmp');
|
||||
const pify = require('pify');
|
||||
const archiver = require('archiver');
|
||||
|
|
@ -1140,12 +1141,13 @@ function getMessageKey(message) {
|
|||
const sourceDevice = message.sourceDevice || 1;
|
||||
return `${source}.${sourceDevice} ${message.timestamp}`;
|
||||
}
|
||||
function loadMessagesLookup(db) {
|
||||
return window.Signal.Data.getAllMessageIds({
|
||||
async function loadMessagesLookup(db) {
|
||||
const array = await window.Signal.Data.getAllMessageIds({
|
||||
db,
|
||||
getMessageKey,
|
||||
handleDOMException: Whisper.Database.handleDOMException,
|
||||
});
|
||||
return fromPairs(map(array, item => [item, true]));
|
||||
}
|
||||
|
||||
function getConversationKey(conversation) {
|
||||
|
|
|
|||
|
|
@ -1,75 +1,233 @@
|
|||
/* global window */
|
||||
/* global window, setTimeout */
|
||||
|
||||
const electron = require('electron');
|
||||
const { forEach, isFunction, isObject } = require('lodash');
|
||||
|
||||
const { deferredToPromise } = require('./deferred_to_promise');
|
||||
const MessageType = require('./types/message');
|
||||
|
||||
// calls to search for:
|
||||
const { ipcRenderer } = electron;
|
||||
|
||||
// We listen to a lot of events on ipcRenderer, often on the same channel. This prevents
|
||||
// any warnings that might be sent to the console in that case.
|
||||
ipcRenderer.setMaxListeners(0);
|
||||
|
||||
// calls to search for when finding functions to convert:
|
||||
// .fetch(
|
||||
// .save(
|
||||
// .destroy(
|
||||
|
||||
async function saveMessage(data, { Message }) {
|
||||
const message = new Message(data);
|
||||
await deferredToPromise(message.save());
|
||||
return message.id;
|
||||
const SQL_CHANNEL_KEY = 'sql-channel';
|
||||
const ERASE_SQL_KEY = 'erase-sql-key';
|
||||
const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
|
||||
|
||||
const _jobs = Object.create(null);
|
||||
const _DEBUG = false;
|
||||
let _jobCounter = 0;
|
||||
|
||||
const channels = {};
|
||||
|
||||
module.exports = {
|
||||
_jobs,
|
||||
_cleanData,
|
||||
|
||||
close,
|
||||
removeDB,
|
||||
|
||||
saveMessage,
|
||||
saveMessages,
|
||||
removeMessage,
|
||||
getUnreadByConversation,
|
||||
|
||||
removeAllMessagesInConversation,
|
||||
|
||||
getMessageBySender,
|
||||
getMessageById,
|
||||
getAllMessageIds,
|
||||
getMessagesBySentAt,
|
||||
getExpiredMessages,
|
||||
getNextExpiringMessage,
|
||||
getMessagesByConversation,
|
||||
|
||||
getAllUnprocessed,
|
||||
getUnprocessedById,
|
||||
saveUnprocessed,
|
||||
saveUnprocesseds,
|
||||
updateUnprocessed,
|
||||
removeUnprocessed,
|
||||
removeAllUnprocessed,
|
||||
|
||||
removeAll,
|
||||
removeOtherData,
|
||||
|
||||
// Returning plain JSON
|
||||
getMessagesNeedingUpgrade,
|
||||
getLegacyMessagesNeedingUpgrade,
|
||||
getMessagesWithVisualMediaAttachments,
|
||||
getMessagesWithFileAttachments,
|
||||
};
|
||||
|
||||
// When IPC arguments are prepared for the cross-process send, they are JSON.stringified.
|
||||
// We can't send ArrayBuffers or BigNumbers (what we get from proto library for dates).
|
||||
function _cleanData(data) {
|
||||
const keys = Object.keys(data);
|
||||
for (let index = 0, max = keys.length; index < max; index += 1) {
|
||||
const key = keys[index];
|
||||
const value = data[key];
|
||||
|
||||
if (value === null || value === undefined) {
|
||||
// eslint-disable-next-line no-continue
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isFunction(value.toNumber)) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
data[key] = value.toNumber();
|
||||
} else if (Array.isArray(value)) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
data[key] = value.map(item => _cleanData(item));
|
||||
} else if (isObject(value)) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
data[key] = _cleanData(value);
|
||||
} else if (
|
||||
typeof value !== 'string' &&
|
||||
typeof value !== 'number' &&
|
||||
typeof value !== 'boolean'
|
||||
) {
|
||||
window.log.info(`_cleanData: key ${key} had type ${typeof value}`);
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function _makeJob(fnName) {
|
||||
_jobCounter += 1;
|
||||
const id = _jobCounter;
|
||||
|
||||
_jobs[id] = {
|
||||
fnName,
|
||||
};
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
function _updateJob(id, data) {
|
||||
const { resolve, reject } = data;
|
||||
|
||||
_jobs[id] = {
|
||||
..._jobs[id],
|
||||
...data,
|
||||
resolve: value => {
|
||||
_removeJob(id);
|
||||
return resolve(value);
|
||||
},
|
||||
reject: error => {
|
||||
_removeJob(id);
|
||||
return reject(error);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function _removeJob(id) {
|
||||
if (_DEBUG) {
|
||||
_jobs[id].complete = true;
|
||||
} else {
|
||||
delete _jobs[id];
|
||||
}
|
||||
}
|
||||
|
||||
function _getJob(id) {
|
||||
return _jobs[id];
|
||||
}
|
||||
|
||||
ipcRenderer.on(
|
||||
`${SQL_CHANNEL_KEY}-done`,
|
||||
(event, jobId, errorForDisplay, result) => {
|
||||
const job = _getJob(jobId);
|
||||
if (!job) {
|
||||
throw new Error(
|
||||
`Received job reply to job ${jobId}, but did not have it in our registry!`
|
||||
);
|
||||
}
|
||||
|
||||
const { resolve, reject, fnName } = job;
|
||||
|
||||
if (errorForDisplay) {
|
||||
return reject(
|
||||
new Error(`Error calling channel ${fnName}: ${errorForDisplay}`)
|
||||
);
|
||||
}
|
||||
|
||||
return resolve(result);
|
||||
}
|
||||
);
|
||||
|
||||
function makeChannel(fnName) {
|
||||
channels[fnName] = (...args) => {
|
||||
const jobId = _makeJob(fnName);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
ipcRenderer.send(SQL_CHANNEL_KEY, jobId, fnName, ...args);
|
||||
|
||||
_updateJob(jobId, {
|
||||
resolve,
|
||||
reject,
|
||||
args: _DEBUG ? args : null,
|
||||
});
|
||||
|
||||
setTimeout(
|
||||
() => resolve(new Error(`Request to ${fnName} timed out`)),
|
||||
5000
|
||||
);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
forEach(module.exports, fn => {
|
||||
if (isFunction(fn)) {
|
||||
makeChannel(fn.name);
|
||||
}
|
||||
});
|
||||
|
||||
// Note: will need to restart the app after calling this, to set up afresh
|
||||
async function close() {
|
||||
await channels.close();
|
||||
}
|
||||
|
||||
// Note: will need to restart the app after calling this, to set up afresh
|
||||
async function removeDB() {
|
||||
await channels.removeDB();
|
||||
}
|
||||
|
||||
async function saveMessage(data, { forceSave } = {}) {
|
||||
const id = await channels.saveMessage(_cleanData(data), { forceSave });
|
||||
return id;
|
||||
}
|
||||
|
||||
async function saveMessages(arrayOfMessages, { forceSave } = {}) {
|
||||
await channels.saveMessages(_cleanData(arrayOfMessages), { forceSave });
|
||||
}
|
||||
|
||||
async function removeMessage(id, { Message }) {
|
||||
const message = await getMessageById(id, { Message });
|
||||
|
||||
// Note: It's important to have a fully database-hydrated model to delete here because
|
||||
// it needs to delete all associated on-disk files along with the database delete.
|
||||
if (message) {
|
||||
await deferredToPromise(message.destroy());
|
||||
await channels.removeMessage(id);
|
||||
const model = new Message(message);
|
||||
await model.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
async function getMessageById(id, { Message }) {
|
||||
const message = new Message({ id });
|
||||
try {
|
||||
await deferredToPromise(message.fetch());
|
||||
return message;
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
const message = await channels.getMessageById(id);
|
||||
return new Message(message);
|
||||
}
|
||||
|
||||
async function getAllMessageIds({ db, handleDOMException, getMessageKey }) {
|
||||
const lookup = Object.create(null);
|
||||
const storeName = 'messages';
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = db.transaction(storeName, 'readwrite');
|
||||
transaction.onerror = () => {
|
||||
handleDOMException(
|
||||
`assembleLookup(${storeName}) transaction error`,
|
||||
transaction.error,
|
||||
reject
|
||||
);
|
||||
};
|
||||
transaction.oncomplete = () => {
|
||||
// not really very useful - fires at unexpected times
|
||||
};
|
||||
|
||||
const store = transaction.objectStore(storeName);
|
||||
const request = store.openCursor();
|
||||
request.onerror = () => {
|
||||
handleDOMException(
|
||||
`assembleLookup(${storeName}) request error`,
|
||||
request.error,
|
||||
reject
|
||||
);
|
||||
};
|
||||
request.onsuccess = event => {
|
||||
const cursor = event.target.result;
|
||||
if (cursor && cursor.value) {
|
||||
lookup[getMessageKey(cursor.value)] = true;
|
||||
cursor.continue();
|
||||
} else {
|
||||
window.log.info(`Done creating ${storeName} lookup`);
|
||||
resolve(lookup);
|
||||
}
|
||||
};
|
||||
});
|
||||
async function getAllMessageIds() {
|
||||
const ids = await channels.getAllMessageIds();
|
||||
return ids;
|
||||
}
|
||||
|
||||
async function getMessageBySender(
|
||||
|
|
@ -77,186 +235,155 @@ async function getMessageBySender(
|
|||
{ source, sourceDevice, sent_at },
|
||||
{ Message }
|
||||
) {
|
||||
const fetcher = new Message();
|
||||
const options = {
|
||||
index: {
|
||||
name: 'unique',
|
||||
// eslint-disable-next-line camelcase
|
||||
value: [source, sourceDevice, sent_at],
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
await deferredToPromise(fetcher.fetch(options));
|
||||
if (fetcher.get('id')) {
|
||||
return fetcher;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error) {
|
||||
const messages = await channels.getMessageBySender({
|
||||
source,
|
||||
sourceDevice,
|
||||
sent_at,
|
||||
});
|
||||
if (!messages || !messages.length) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Message(messages[0]);
|
||||
}
|
||||
|
||||
async function getUnreadByConversation(conversationId, { MessageCollection }) {
|
||||
const messages = new MessageCollection();
|
||||
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
index: {
|
||||
// 'unread' index
|
||||
name: 'unread',
|
||||
lower: [conversationId],
|
||||
upper: [conversationId, Number.MAX_VALUE],
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
return messages;
|
||||
const messages = await channels.getUnreadByConversation(conversationId);
|
||||
return new MessageCollection(messages);
|
||||
}
|
||||
|
||||
async function getMessagesByConversation(
|
||||
conversationId,
|
||||
{ limit = 100, receivedAt = Number.MAX_VALUE, MessageCollection }
|
||||
) {
|
||||
const messages = new MessageCollection();
|
||||
|
||||
const options = {
|
||||
const messages = await channels.getMessagesByConversation(conversationId, {
|
||||
limit,
|
||||
index: {
|
||||
// 'conversation' index on [conversationId, received_at]
|
||||
name: 'conversation',
|
||||
lower: [conversationId],
|
||||
upper: [conversationId, receivedAt],
|
||||
order: 'desc',
|
||||
// SELECT messages WHERE conversationId = this.id ORDER
|
||||
// received_at DESC
|
||||
},
|
||||
};
|
||||
await deferredToPromise(messages.fetch(options));
|
||||
receivedAt,
|
||||
});
|
||||
|
||||
return messages;
|
||||
return new MessageCollection(messages);
|
||||
}
|
||||
|
||||
async function removeAllMessagesInConversation(
|
||||
conversationId,
|
||||
{ MessageCollection }
|
||||
) {
|
||||
const messages = new MessageCollection();
|
||||
|
||||
let loaded;
|
||||
let messages;
|
||||
do {
|
||||
// Yes, we really want the await in the loop. We're deleting 100 at a
|
||||
// time so we don't use too much memory.
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
limit: 100,
|
||||
index: {
|
||||
// 'conversation' index on [conversationId, received_at]
|
||||
name: 'conversation',
|
||||
lower: [conversationId],
|
||||
upper: [conversationId, Number.MAX_VALUE],
|
||||
},
|
||||
})
|
||||
);
|
||||
messages = await getMessagesByConversation(conversationId, {
|
||||
limit: 100,
|
||||
MessageCollection,
|
||||
});
|
||||
|
||||
loaded = messages.models;
|
||||
messages.reset([]);
|
||||
if (!messages.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ids = messages.map(message => message.id);
|
||||
|
||||
// Note: It's very important that these models are fully hydrated because
|
||||
// we need to delete all associated on-disk files along with the database delete.
|
||||
loaded.map(message => message.destroy());
|
||||
} while (loaded.length > 0);
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await Promise.all(messages.map(message => message.cleanup()));
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await channels.removeMessage(ids);
|
||||
} while (messages.length > 0);
|
||||
}
|
||||
|
||||
async function getMessagesBySentAt(sentAt, { MessageCollection }) {
|
||||
const messages = new MessageCollection();
|
||||
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
index: {
|
||||
// 'receipt' index on sent_at
|
||||
name: 'receipt',
|
||||
only: sentAt,
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
return messages;
|
||||
const messages = await channels.getMessagesBySentAt(sentAt);
|
||||
return new MessageCollection(messages);
|
||||
}
|
||||
|
||||
async function getExpiredMessages({ MessageCollection }) {
|
||||
window.log.info('Load expired messages');
|
||||
const messages = new MessageCollection();
|
||||
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
conditions: {
|
||||
expires_at: {
|
||||
$lte: Date.now(),
|
||||
},
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
return messages;
|
||||
const messages = await channels.getExpiredMessages();
|
||||
return new MessageCollection(messages);
|
||||
}
|
||||
|
||||
async function getNextExpiringMessage({ MessageCollection }) {
|
||||
const messages = new MessageCollection();
|
||||
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
limit: 1,
|
||||
index: {
|
||||
name: 'expires_at',
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
return messages;
|
||||
const messages = await channels.getNextExpiringMessage();
|
||||
return new MessageCollection(messages);
|
||||
}
|
||||
|
||||
async function saveUnprocessed(data, { Unprocessed }) {
|
||||
const unprocessed = new Unprocessed(data);
|
||||
return deferredToPromise(unprocessed.save());
|
||||
async function getAllUnprocessed() {
|
||||
return channels.getAllUnprocessed();
|
||||
}
|
||||
|
||||
async function getAllUnprocessed({ UnprocessedCollection }) {
|
||||
const collection = new UnprocessedCollection();
|
||||
await deferredToPromise(collection.fetch());
|
||||
return collection.map(model => model.attributes);
|
||||
async function getUnprocessedById(id, { Unprocessed }) {
|
||||
const unprocessed = await channels.getUnprocessedById(id);
|
||||
return new Unprocessed(unprocessed);
|
||||
}
|
||||
|
||||
async function updateUnprocessed(id, updates, { Unprocessed }) {
|
||||
const unprocessed = new Unprocessed({
|
||||
id,
|
||||
async function saveUnprocessed(data, { forceSave } = {}) {
|
||||
const id = await channels.saveUnprocessed(_cleanData(data), { forceSave });
|
||||
return id;
|
||||
}
|
||||
|
||||
async function saveUnprocesseds(arrayOfUnprocessed, { forceSave } = {}) {
|
||||
await channels.saveUnprocesseds(_cleanData(arrayOfUnprocessed), {
|
||||
forceSave,
|
||||
});
|
||||
|
||||
await deferredToPromise(unprocessed.fetch());
|
||||
|
||||
unprocessed.set(updates);
|
||||
await saveUnprocessed(unprocessed.attributes, { Unprocessed });
|
||||
}
|
||||
|
||||
async function removeUnprocessed(id, { Unprocessed }) {
|
||||
const unprocessed = new Unprocessed({
|
||||
id,
|
||||
});
|
||||
async function updateUnprocessed(id, updates) {
|
||||
const existing = await channels.getUnprocessedById(id);
|
||||
if (!existing) {
|
||||
throw new Error(`Unprocessed id ${id} does not exist in the database!`);
|
||||
}
|
||||
const toSave = {
|
||||
...existing,
|
||||
...updates,
|
||||
};
|
||||
|
||||
await deferredToPromise(unprocessed.destroy());
|
||||
await saveUnprocessed(toSave);
|
||||
}
|
||||
|
||||
async function removeUnprocessed(id) {
|
||||
await channels.removeUnprocessed(id);
|
||||
}
|
||||
|
||||
async function removeAllUnprocessed() {
|
||||
// erase everything in unprocessed table
|
||||
await channels.removeAllUnprocessed();
|
||||
}
|
||||
|
||||
async function removeAll() {
|
||||
// erase everything in the database
|
||||
await channels.removeAll();
|
||||
}
|
||||
|
||||
async function getMessagesNeedingUpgrade(
|
||||
// Note: will need to restart the app after calling this, to set up afresh
|
||||
async function removeOtherData() {
|
||||
await Promise.all([
|
||||
callChannel(ERASE_SQL_KEY),
|
||||
callChannel(ERASE_ATTACHMENTS_KEY),
|
||||
]);
|
||||
}
|
||||
|
||||
async function callChannel(name) {
|
||||
return new Promise((resolve, reject) => {
|
||||
ipcRenderer.send(name);
|
||||
ipcRenderer.once(`${name}-done`, (event, error) => {
|
||||
if (error) {
|
||||
return reject(error);
|
||||
}
|
||||
|
||||
return resolve();
|
||||
});
|
||||
|
||||
setTimeout(
|
||||
() => reject(new Error(`callChannel call to ${name} timed out`)),
|
||||
5000
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Functions below here return JSON
|
||||
|
||||
async function getLegacyMessagesNeedingUpgrade(
|
||||
limit,
|
||||
{ MessageCollection, maxVersion = MessageType.CURRENT_SCHEMA_VERSION }
|
||||
) {
|
||||
|
|
@ -278,75 +405,28 @@ async function getMessagesNeedingUpgrade(
|
|||
return models.map(model => model.toJSON());
|
||||
}
|
||||
|
||||
async function getMessagesNeedingUpgrade(
|
||||
limit,
|
||||
{ maxVersion = MessageType.CURRENT_SCHEMA_VERSION }
|
||||
) {
|
||||
const messages = await channels.getMessagesNeedingUpgrade(limit, {
|
||||
maxVersion,
|
||||
});
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
async function getMessagesWithVisualMediaAttachments(
|
||||
conversationId,
|
||||
{ limit, MessageCollection }
|
||||
{ limit }
|
||||
) {
|
||||
const messages = new MessageCollection();
|
||||
const lowerReceivedAt = 0;
|
||||
const upperReceivedAt = Number.MAX_VALUE;
|
||||
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
limit,
|
||||
index: {
|
||||
name: 'hasVisualMediaAttachments',
|
||||
lower: [conversationId, lowerReceivedAt, 1],
|
||||
upper: [conversationId, upperReceivedAt, 1],
|
||||
order: 'desc',
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
return messages.models.map(model => model.toJSON());
|
||||
return channels.getMessagesWithVisualMediaAttachments(conversationId, {
|
||||
limit,
|
||||
});
|
||||
}
|
||||
|
||||
async function getMessagesWithFileAttachments(
|
||||
conversationId,
|
||||
{ limit, MessageCollection }
|
||||
) {
|
||||
const messages = new MessageCollection();
|
||||
const lowerReceivedAt = 0;
|
||||
const upperReceivedAt = Number.MAX_VALUE;
|
||||
|
||||
await deferredToPromise(
|
||||
messages.fetch({
|
||||
limit,
|
||||
index: {
|
||||
name: 'hasFileAttachments',
|
||||
lower: [conversationId, lowerReceivedAt, 1],
|
||||
upper: [conversationId, upperReceivedAt, 1],
|
||||
order: 'desc',
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
return messages.models.map(model => model.toJSON());
|
||||
async function getMessagesWithFileAttachments(conversationId, { limit }) {
|
||||
return channels.getMessagesWithFileAttachments(conversationId, {
|
||||
limit,
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
saveMessage,
|
||||
removeMessage,
|
||||
getUnreadByConversation,
|
||||
removeAllMessagesInConversation,
|
||||
getMessageBySender,
|
||||
getMessageById,
|
||||
getAllMessageIds,
|
||||
getMessagesBySentAt,
|
||||
getExpiredMessages,
|
||||
getNextExpiringMessage,
|
||||
getMessagesByConversation,
|
||||
|
||||
getAllUnprocessed,
|
||||
saveUnprocessed,
|
||||
updateUnprocessed,
|
||||
removeUnprocessed,
|
||||
removeAllUnprocessed,
|
||||
|
||||
removeAll,
|
||||
|
||||
// Returning plain JSON
|
||||
getMessagesNeedingUpgrade,
|
||||
getMessagesWithVisualMediaAttachments,
|
||||
getMessagesWithFileAttachments,
|
||||
};
|
||||
|
|
|
|||
167
js/modules/migrate_to_sql.js
Normal file
167
js/modules/migrate_to_sql.js
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
/* global window, IDBKeyRange */
|
||||
|
||||
const { includes, isFunction, isString, last } = require('lodash');
|
||||
const { saveMessages, saveUnprocesseds } = require('./data');
|
||||
const {
|
||||
getMessageExportLastIndex,
|
||||
setMessageExportLastIndex,
|
||||
getUnprocessedExportLastIndex,
|
||||
setUnprocessedExportLastIndex,
|
||||
} = require('./settings');
|
||||
|
||||
module.exports = {
|
||||
migrateToSQL,
|
||||
};
|
||||
|
||||
async function migrateToSQL({ db, clearStores, handleDOMException }) {
|
||||
if (!db) {
|
||||
throw new Error('Need db for IndexedDB connection!');
|
||||
}
|
||||
if (!isFunction(clearStores)) {
|
||||
throw new Error('Need clearStores function!');
|
||||
}
|
||||
if (!isFunction(handleDOMException)) {
|
||||
throw new Error('Need handleDOMException function!');
|
||||
}
|
||||
|
||||
window.log.info('migrateToSQL: start');
|
||||
|
||||
let lastIndex = await getMessageExportLastIndex(db);
|
||||
let complete = false;
|
||||
|
||||
while (!complete) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const status = await migrateStoreToSQLite({
|
||||
db,
|
||||
save: saveMessages,
|
||||
storeName: 'messages',
|
||||
handleDOMException,
|
||||
lastIndex,
|
||||
});
|
||||
|
||||
({ complete, lastIndex } = status);
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await setMessageExportLastIndex(db, lastIndex);
|
||||
}
|
||||
window.log.info('migrateToSQL: migrate of messages complete');
|
||||
|
||||
lastIndex = await getUnprocessedExportLastIndex(db);
|
||||
complete = false;
|
||||
|
||||
while (!complete) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const status = await migrateStoreToSQLite({
|
||||
db,
|
||||
save: saveUnprocesseds,
|
||||
storeName: 'unprocessed',
|
||||
handleDOMException,
|
||||
lastIndex,
|
||||
});
|
||||
|
||||
({ complete, lastIndex } = status);
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await setUnprocessedExportLastIndex(db, lastIndex);
|
||||
}
|
||||
window.log.info('migrateToSQL: migrate of unprocessed complete');
|
||||
|
||||
await clearStores(['messages', 'unprocessed']);
|
||||
|
||||
window.log.info('migrateToSQL: complete');
|
||||
}
|
||||
|
||||
async function migrateStoreToSQLite({
|
||||
db,
|
||||
save,
|
||||
storeName,
|
||||
handleDOMException,
|
||||
lastIndex = null,
|
||||
batchSize = 20,
|
||||
}) {
|
||||
if (!db) {
|
||||
throw new Error('Need db for IndexedDB connection!');
|
||||
}
|
||||
if (!isFunction(save)) {
|
||||
throw new Error('Need save function!');
|
||||
}
|
||||
if (!isString(storeName)) {
|
||||
throw new Error('Need storeName!');
|
||||
}
|
||||
if (!isFunction(handleDOMException)) {
|
||||
throw new Error('Need handleDOMException for error handling!');
|
||||
}
|
||||
|
||||
if (!includes(db.objectStoreNames, storeName)) {
|
||||
return {
|
||||
complete: true,
|
||||
count: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const queryPromise = new Promise((resolve, reject) => {
|
||||
const items = [];
|
||||
const transaction = db.transaction(storeName, 'readonly');
|
||||
transaction.onerror = () => {
|
||||
handleDOMException(
|
||||
'migrateToSQLite transaction error',
|
||||
transaction.error,
|
||||
reject
|
||||
);
|
||||
};
|
||||
transaction.oncomplete = () => {};
|
||||
|
||||
const store = transaction.objectStore(storeName);
|
||||
const excludeLowerBound = true;
|
||||
const range = lastIndex
|
||||
? IDBKeyRange.lowerBound(lastIndex, excludeLowerBound)
|
||||
: undefined;
|
||||
const request = store.openCursor(range);
|
||||
request.onerror = () => {
|
||||
handleDOMException(
|
||||
'migrateToSQLite: request error',
|
||||
request.error,
|
||||
reject
|
||||
);
|
||||
};
|
||||
request.onsuccess = event => {
|
||||
const cursor = event.target.result;
|
||||
|
||||
if (!cursor || !cursor.value) {
|
||||
return resolve({
|
||||
complete: true,
|
||||
items,
|
||||
});
|
||||
}
|
||||
|
||||
const item = cursor.value;
|
||||
items.push(item);
|
||||
|
||||
if (items.length >= batchSize) {
|
||||
return resolve({
|
||||
complete: false,
|
||||
items,
|
||||
});
|
||||
}
|
||||
|
||||
return cursor.continue();
|
||||
};
|
||||
});
|
||||
|
||||
const { items, complete } = await queryPromise;
|
||||
|
||||
if (items.length) {
|
||||
// We need to pass forceSave parameter, because these items already have an
|
||||
// id key. Normally, this call would be interpreted as an update request.
|
||||
await save(items, { forceSave: true });
|
||||
}
|
||||
|
||||
const lastItem = last(items);
|
||||
const id = lastItem ? lastItem.id : null;
|
||||
|
||||
return {
|
||||
complete,
|
||||
count: items.length,
|
||||
lastIndex: id,
|
||||
};
|
||||
}
|
||||
|
|
@ -3,25 +3,34 @@ const { isObject, isString } = require('lodash');
|
|||
const ITEMS_STORE_NAME = 'items';
|
||||
const LAST_PROCESSED_INDEX_KEY = 'attachmentMigration_lastProcessedIndex';
|
||||
const IS_MIGRATION_COMPLETE_KEY = 'attachmentMigration_isComplete';
|
||||
const MESSAGE_LAST_INDEX_KEY = 'sqlMigration_messageLastIndex';
|
||||
const UNPROCESSED_LAST_INDEX_KEY = 'sqlMigration_unprocessedLastIndex';
|
||||
|
||||
// Public API
|
||||
exports.READ_RECEIPT_CONFIGURATION_SYNC = 'read-receipt-configuration-sync';
|
||||
|
||||
exports.getAttachmentMigrationLastProcessedIndex = connection =>
|
||||
exports._getItem(connection, LAST_PROCESSED_INDEX_KEY);
|
||||
|
||||
exports.setAttachmentMigrationLastProcessedIndex = (connection, value) =>
|
||||
exports._setItem(connection, LAST_PROCESSED_INDEX_KEY, value);
|
||||
|
||||
exports.deleteAttachmentMigrationLastProcessedIndex = connection =>
|
||||
exports._deleteItem(connection, LAST_PROCESSED_INDEX_KEY);
|
||||
|
||||
exports.isAttachmentMigrationComplete = async connection =>
|
||||
Boolean(await exports._getItem(connection, IS_MIGRATION_COMPLETE_KEY));
|
||||
|
||||
exports.markAttachmentMigrationComplete = connection =>
|
||||
exports._setItem(connection, IS_MIGRATION_COMPLETE_KEY, true);
|
||||
|
||||
exports.getMessageExportLastIndex = connection =>
|
||||
exports._getItem(connection, MESSAGE_LAST_INDEX_KEY);
|
||||
exports.setMessageExportLastIndex = (connection, lastIndex) =>
|
||||
exports._setItem(connection, MESSAGE_LAST_INDEX_KEY, lastIndex);
|
||||
|
||||
exports.getUnprocessedExportLastIndex = connection =>
|
||||
exports._getItem(connection, UNPROCESSED_LAST_INDEX_KEY);
|
||||
exports.setUnprocessedExportLastIndex = (connection, lastIndex) =>
|
||||
exports._setItem(connection, UNPROCESSED_LAST_INDEX_KEY, lastIndex);
|
||||
|
||||
// Private API
|
||||
exports._getItem = (connection, key) => {
|
||||
if (!isObject(connection)) {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ const OS = require('../../ts/OS');
|
|||
const Settings = require('./settings');
|
||||
const Startup = require('./startup');
|
||||
const Util = require('../../ts/util');
|
||||
const { migrateToSQL } = require('./migrate_to_sql');
|
||||
|
||||
// Components
|
||||
const {
|
||||
|
|
@ -110,6 +111,7 @@ function initializeMigrations({
|
|||
const attachmentsPath = getPath(userDataPath);
|
||||
const readAttachmentData = createReader(attachmentsPath);
|
||||
const loadAttachmentData = Type.loadData(readAttachmentData);
|
||||
const loadQuoteData = MessageType.loadQuoteData(readAttachmentData);
|
||||
const getAbsoluteAttachmentPath = createAbsolutePathGetter(attachmentsPath);
|
||||
const deleteOnDisk = Attachments.createDeleter(attachmentsPath);
|
||||
|
||||
|
|
@ -122,6 +124,7 @@ function initializeMigrations({
|
|||
getAbsoluteAttachmentPath,
|
||||
getPlaceholderMigrations,
|
||||
loadAttachmentData,
|
||||
loadQuoteData,
|
||||
loadMessage: MessageType.createAttachmentLoader(loadAttachmentData),
|
||||
Migrations0DatabaseWithAttachmentData,
|
||||
Migrations1DatabaseWithoutAttachmentData,
|
||||
|
|
@ -222,5 +225,6 @@ exports.setup = (options = {}) => {
|
|||
Util,
|
||||
Views,
|
||||
Workflow,
|
||||
migrateToSQL,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ exports._mapAttachments = upgradeAttachment => async (message, context) => {
|
|||
const upgradeWithContext = attachment =>
|
||||
upgradeAttachment(attachment, context);
|
||||
const attachments = await Promise.all(
|
||||
message.attachments.map(upgradeWithContext)
|
||||
(message.attachments || []).map(upgradeWithContext)
|
||||
);
|
||||
return Object.assign({}, message, { attachments });
|
||||
};
|
||||
|
|
@ -356,7 +356,9 @@ exports.upgradeSchema = async (
|
|||
|
||||
exports.createAttachmentLoader = loadAttachmentData => {
|
||||
if (!isFunction(loadAttachmentData)) {
|
||||
throw new TypeError('`loadAttachmentData` is required');
|
||||
throw new TypeError(
|
||||
'createAttachmentLoader: loadAttachmentData is required'
|
||||
);
|
||||
}
|
||||
|
||||
return async message =>
|
||||
|
|
@ -367,6 +369,36 @@ exports.createAttachmentLoader = loadAttachmentData => {
|
|||
});
|
||||
};
|
||||
|
||||
exports.loadQuoteData = loadAttachmentData => {
|
||||
if (!isFunction(loadAttachmentData)) {
|
||||
throw new TypeError('loadQuoteData: loadAttachmentData is required');
|
||||
}
|
||||
|
||||
return async quote => {
|
||||
if (!quote) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...quote,
|
||||
attachments: await Promise.all(
|
||||
(quote.attachments || []).map(async attachment => {
|
||||
const { thumbnail } = attachment;
|
||||
|
||||
if (!thumbnail || !thumbnail.path) {
|
||||
return attachment;
|
||||
}
|
||||
|
||||
return {
|
||||
...attachment,
|
||||
thumbnail: await loadAttachmentData(thumbnail),
|
||||
};
|
||||
})
|
||||
),
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => {
|
||||
if (!isFunction(deleteAttachmentData)) {
|
||||
throw new TypeError(
|
||||
|
|
@ -392,7 +424,7 @@ exports.deleteAllExternalFiles = ({ deleteAttachmentData, deleteOnDisk }) => {
|
|||
quote.attachments.map(async attachment => {
|
||||
const { thumbnail } = attachment;
|
||||
|
||||
if (thumbnail.path) {
|
||||
if (thumbnail && thumbnail.path) {
|
||||
await deleteOnDisk(thumbnail.path);
|
||||
}
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue