mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Improve message download performance

Browse source
This commit is contained in:
Scott Nonnenberg 2019-09-26 12:56:31 -07:00
parent 957f6f6474
commit 0c09f9620f
32 changed files with 906 additions and 633 deletions
Download patch file Download diff file Expand all files Collapse all files

11
app/logging.js
View file

@ -69,6 +69,17 @@ function initialize() {
});
});
ipc.on('batch-log', (first, batch) => {
batch.forEach(item => {
logger[item.level](
{
time: new Date(item.timestamp),
},
item.logText
);
});
});
ipc.on('fetch-log', event => {
fetch(logPath).then(
data => {

63
app/sql.js
View file

@ -19,10 +19,6 @@ const {
pick,
} = require('lodash');
// To get long stack traces
// https://github.com/mapbox/node-sqlite3/wiki/API#sqlite3verbose
sql.verbose();
module.exports = {
initialize,
close,
@ -58,6 +54,7 @@ module.exports = {
removeAllItems,
createOrUpdateSession,
createOrUpdateSessions,
getSessionById,
getSessionsByNumber,
bulkAddSessions,
@ -71,6 +68,7 @@ module.exports = {
saveConversations,
getConversationById,
updateConversation,
updateConversations,
removeConversation,
getAllConversations,
getAllConversationIds,
@ -105,6 +103,7 @@ module.exports = {
saveUnprocessed,
updateUnprocessedAttempts,
updateUnprocessedWithData,
updateUnprocessedsWithData,
getUnprocessedById,
saveUnprocesseds,
removeUnprocessed,
@ -1259,10 +1258,20 @@ async function initialize({ configDir, key, messages }) {
promisified = await openAndSetUpSQLCipher(filePath, { key });
// promisified.on('trace', async statement => {
// if (!db || statement.startsWith('--')) {
// console._log(statement);
// if (
// !db ||
// statement.startsWith('--') ||
// statement.includes('COMMIT') ||
// statement.includes('BEGIN') ||
// statement.includes('ROLLBACK')
// ) {
// return;
// }
// // Note that this causes problems when attempting to commit transactions - this
// // statement is running, and we get at SQLITE_BUSY error. So we delay.
// await new Promise(resolve => setTimeout(resolve, 1000));
// const data = await db.get(`EXPLAIN QUERY PLAN ${statement}`);
// console._log(`EXPLAIN QUERY PLAN ${statement}\n`, data && data.detail);
// });
@ -1469,6 +1478,19 @@ async function createOrUpdateSession(data) {
}
);
}
async function createOrUpdateSessions(array) {
await db.run('BEGIN TRANSACTION;');
try {
await Promise.all([...map(array, item => createOrUpdateSession(item))]);
await db.run('COMMIT TRANSACTION;');
} catch (error) {
await db.run('ROLLBACK;');
throw error;
}
}
createOrUpdateSessions.needsSerial = true;
async function getSessionById(id) {
return getById(SESSIONS_TABLE, id);
}
@ -1663,6 +1685,18 @@ async function updateConversation(data) {
}
);
}
async function updateConversations(array) {
await db.run('BEGIN TRANSACTION;');
try {
await Promise.all([...map(array, item => updateConversation(item))]);
await db.run('COMMIT TRANSACTION;');
} catch (error) {
await db.run('ROLLBACK;');
throw error;
}
}
updateConversations.needsSerial = true;
async function removeConversation(id) {
if (!Array.isArray(id)) {
@ -2353,6 +2387,23 @@ async function updateUnprocessedWithData(id, data = {}) {
}
);
}
async function updateUnprocessedsWithData(arrayOfUnprocessed) {
await db.run('BEGIN TRANSACTION;');
try {
await Promise.all([
...map(arrayOfUnprocessed, ({ id, data }) =>
updateUnprocessedWithData(id, data)
),
]);
await db.run('COMMIT TRANSACTION;');
} catch (error) {
await db.run('ROLLBACK;');
throw error;
}
}
updateUnprocessedsWithData.needsSerial = true;
async function getUnprocessedById(id) {
const row = await db.get('SELECT * FROM unprocessed WHERE id = $id;', {

123
app/sql_channel.js
View file

@ -27,6 +27,75 @@ function makeNewMultipleQueue() {
return multipleQueue;
}
function makeSQLJob(fn, callName, jobId, args) {
// console.log(`Job ${jobId} (${callName}) queued`);
return async () => {
// const start = Date.now();
// console.log(`Job ${jobId} (${callName}) started`);
const result = await fn(...args);
// const end = Date.now();
// console.log(`Job ${jobId} (${callName}) succeeded in ${end - start}ms`);
return result;
};
}
async function handleCall(callName, jobId, args) {
const fn = sql[callName];
if (!fn) {
throw new Error(`sql channel: ${callName} is not an available function`);
}
let result;
// We queue here to keep multi-query operations atomic. Without it, any multistage
// data operation (even within a BEGIN/COMMIT) can become interleaved, since all
// requests share one database connection.
// A needsSerial method must be run in our single concurrency queue.
if (fn.needsSerial) {
if (singleQueue) {
result = await singleQueue.add(makeSQLJob(fn, callName, jobId, args));
} else if (multipleQueue) {
makeNewSingleQueue();
singleQueue.add(() => multipleQueue.onIdle());
multipleQueue = null;
result = await singleQueue.add(makeSQLJob(fn, callName, jobId, args));
} else {
makeNewSingleQueue();
result = await singleQueue.add(makeSQLJob(fn, callName, jobId, args));
}
} else {
// The request can be parallelized. To keep the same structure as the above block
// we force this section into the 'lonely if' pattern.
// eslint-disable-next-line no-lonely-if
if (multipleQueue) {
result = await multipleQueue.add(makeSQLJob(fn, callName, jobId, args));
} else if (singleQueue) {
makeNewMultipleQueue();
multipleQueue.pause();
const multipleQueueRef = multipleQueue;
const singleQueueRef = singleQueue;
singleQueue = null;
const promise = multipleQueueRef.add(
makeSQLJob(fn, callName, jobId, args)
);
await singleQueueRef.onIdle();
multipleQueueRef.start();
result = await promise;
} else {
makeNewMultipleQueue();
result = await multipleQueue.add(makeSQLJob(fn, callName, jobId, args));
}
}
return result;
}
function initialize() {
if (initialized) {
throw new Error('sqlChannels: already initialized!');
@ -35,59 +104,7 @@ function initialize() {
ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => {
try {
const fn = sql[callName];
if (!fn) {
throw new Error(
`sql channel: ${callName} is not an available function`
);
}
let result;
// We queue here to keep multi-query operations atomic. Without it, any multistage
// data operation (even within a BEGIN/COMMIT) can become interleaved, since all
// requests share one database connection.
// A needsSerial method must be run in our single concurrency queue.
if (fn.needsSerial) {
if (singleQueue) {
result = await singleQueue.add(() => fn(...args));
} else if (multipleQueue) {
makeNewSingleQueue();
singleQueue.add(() => multipleQueue.onIdle());
multipleQueue = null;
result = await singleQueue.add(() => fn(...args));
} else {
makeNewSingleQueue();
result = await singleQueue.add(() => fn(...args));
}
} else {
// The request can be parallelized. To keep the same structure as the above block
// we force this section into the 'lonely if' pattern.
// eslint-disable-next-line no-lonely-if
if (multipleQueue) {
result = await multipleQueue.add(() => fn(...args));
} else if (singleQueue) {
makeNewMultipleQueue();
multipleQueue.pause();
const multipleQueueRef = multipleQueue;
const singleQueueRef = singleQueue;
singleQueue = null;
const promise = multipleQueueRef.add(() => fn(...args));
await singleQueueRef.onIdle();
multipleQueueRef.start();
result = await promise;
} else {
makeNewMultipleQueue();
result = await multipleQueue.add(() => fn(...args));
}
}
const result = await handleCall(callName, jobId, args);
event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result);
} catch (error) {
const errorForDisplay = error && error.stack ? error.stack : error;