From 37c8c1727f3ac6d6933762054b197dc1c366e752 Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Mon, 5 Apr 2021 15:18:19 -0700 Subject: [PATCH] Types, better-sqlite3, and worker_threads for our sqlite --- ACKNOWLEDGMENTS.md | 52 +- app/sql_channel.js | 94 +- main.js | 36 +- package.json | 27 +- ts/LibSignalStore.ts | 54 +- ts/background.ts | 4 + ts/groups.ts | 4 +- ts/model-types.d.ts | 74 +- ts/models/conversations.ts | 4 +- ts/models/messages.ts | 6 +- ts/sql/Client.ts | 36 +- ts/sql/Interface.ts | 179 +- ts/sql/Queueing.ts | 141 - ts/sql/Server.ts | 4636 +++++++++++++++---------------- ts/sql/main.ts | 113 + ts/sql/mainWorker.ts | 56 + ts/sqlcipher.d.ts | 184 -- ts/state/ducks/conversations.ts | 17 +- ts/state/ducks/search.ts | 11 +- ts/util/combineNames.ts | 2 +- ts/util/lint/exceptions.json | 76 +- ts/util/lint/linter.ts | 2 + ts/views/conversation_view.ts | 10 +- yarn.lock | 126 +- 24 files changed, 2823 insertions(+), 3121 deletions(-) delete mode 100644 ts/sql/Queueing.ts create mode 100644 ts/sql/main.ts create mode 100644 ts/sql/mainWorker.ts delete mode 100644 ts/sqlcipher.d.ts diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 703a50c2fc..e977cab9d3 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -5,34 +5,6 @@ Signal Desktop makes use of the following open source projects. -## @journeyapps/sqlcipher - - Copyright (c) MapBox - All rights reserved. - - Redistribution and use in source and binary forms, with or without modification, - are permitted provided that the following conditions are met: - - - Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. - - Neither the name "MapBox" nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ## @sindresorhus/is MIT License @@ -154,6 +126,30 @@ Signal Desktop makes use of the following open source projects. FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +## better-sqlite3 + + The MIT License (MIT) + + Copyright (c) 2017 Joshua Wise + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ## blob-util Apache License diff --git a/app/sql_channel.js b/app/sql_channel.js index 8e02809e5b..e728efe9a8 100644 --- a/app/sql_channel.js +++ b/app/sql_channel.js @@ -2,13 +2,13 @@ // SPDX-License-Identifier: AGPL-3.0-only const electron = require('electron'); -const Queue = require('p-queue').default; -const sql = require('../ts/sql/Server').default; const { remove: removeUserConfig } = require('./user_config'); const { remove: removeEphemeralConfig } = require('./ephemeral_config'); const { ipcMain } = electron; +let sql; + module.exports = { initialize, }; @@ -18,99 +18,17 @@ let initialized = false; const SQL_CHANNEL_KEY = 'sql-channel'; const ERASE_SQL_KEY = 'erase-sql-key'; -let singleQueue = null; -let multipleQueue = null; - -// Note: we don't want queue timeouts, because delays here are due to in-progress sql -// operations. For example we might try to start a transaction when the prevous isn't -// done, causing that database operation to fail. -function makeNewSingleQueue() { - singleQueue = new Queue({ concurrency: 1 }); - return singleQueue; -} -function makeNewMultipleQueue() { - multipleQueue = new Queue({ concurrency: 10 }); - return multipleQueue; -} - -function makeSQLJob(fn, callName, jobId, args) { - // console.log(`Job ${jobId} (${callName}) queued`); - return async () => { - // const start = Date.now(); - // console.log(`Job ${jobId} (${callName}) started`); - const result = await fn(...args); - // const end = Date.now(); - // console.log(`Job ${jobId} (${callName}) succeeded in ${end - start}ms`); - return result; - }; -} - -async function handleCall(callName, jobId, args) { - const fn = sql[callName]; - if (!fn) { - throw new Error(`sql channel: ${callName} is not an available function`); - } - - let result; - - // We queue here to keep multi-query operations atomic. Without it, any multistage - // data operation (even within a BEGIN/COMMIT) can become interleaved, since all - // requests share one database connection. - - // A needsSerial method must be run in our single concurrency queue. - if (fn.needsSerial) { - if (singleQueue) { - result = await singleQueue.add(makeSQLJob(fn, callName, jobId, args)); - } else if (multipleQueue) { - makeNewSingleQueue(); - - singleQueue.add(() => multipleQueue.onIdle()); - multipleQueue = null; - - result = await singleQueue.add(makeSQLJob(fn, callName, jobId, args)); - } else { - makeNewSingleQueue(); - result = await singleQueue.add(makeSQLJob(fn, callName, jobId, args)); - } - } else { - // The request can be parallelized. To keep the same structure as the above block - // we force this section into the 'lonely if' pattern. - // eslint-disable-next-line no-lonely-if - if (multipleQueue) { - result = await multipleQueue.add(makeSQLJob(fn, callName, jobId, args)); - } else if (singleQueue) { - makeNewMultipleQueue(); - multipleQueue.pause(); - - const multipleQueueRef = multipleQueue; - const singleQueueRef = singleQueue; - - singleQueue = null; - const promise = multipleQueueRef.add( - makeSQLJob(fn, callName, jobId, args) - ); - await singleQueueRef.onIdle(); - - multipleQueueRef.start(); - result = await promise; - } else { - makeNewMultipleQueue(); - result = await multipleQueue.add(makeSQLJob(fn, callName, jobId, args)); - } - } - - return result; -} - -function initialize() { +function initialize(mainSQL) { if (initialized) { throw new Error('sqlChannels: already initialized!'); } initialized = true; + sql = mainSQL; + ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => { try { - const result = await handleCall(callName, jobId, args); + const result = await sql.sqlCall(callName, args); event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result); } catch (error) { const errorForDisplay = error && error.stack ? error.stack : error; diff --git a/main.js b/main.js index 9a59c9ed04..03f5ccc774 100644 --- a/main.js +++ b/main.js @@ -93,7 +93,7 @@ const createTrayIcon = require('./app/tray_icon'); const dockIcon = require('./ts/dock_icon'); const ephemeralConfig = require('./app/ephemeral_config'); const logging = require('./ts/logging/main_process_logging'); -const sql = require('./ts/sql/Server').default; +const { MainSQL } = require('./ts/sql/main'); const sqlChannels = require('./app/sql_channel'); const windowState = require('./app/window_state'); const { createTemplate } = require('./app/menu'); @@ -119,6 +119,8 @@ const { } = require('./ts/types/Settings'); const { Environment } = require('./ts/environment'); +const sql = new MainSQL(); + let appStartInitialSpellcheckSetting = true; const defaultWebPrefs = { @@ -128,7 +130,7 @@ const defaultWebPrefs = { }; async function getSpellCheckSetting() { - const json = await sql.getItemById('spell-check'); + const json = await sql.sqlCall('getItemById', ['spell-check']); // Default to `true` if setting doesn't exist yet if (!json) { @@ -500,6 +502,7 @@ async function createWindow() { if (mainWindow) { mainWindow.readyForShutdown = true; } + await sql.close(); app.quit(); }); @@ -765,8 +768,8 @@ function showSettingsWindow() { async function getIsLinked() { try { - const number = await sql.getItemById('number_id'); - const password = await sql.getItemById('password'); + const number = await sql.sqlCall('getItemById', ['number_id']); + const password = await sql.sqlCall('getItemById', ['password']); return Boolean(number && password); } catch (e) { return false; @@ -1090,7 +1093,7 @@ app.on('ready', async () => { `Database startup error:\n\n${redactAll(error.stack)}` ); } else { - await sql.removeDB(); + await sql.sqlCall('removeDB', []); removeUserConfig(); app.relaunch(); } @@ -1102,14 +1105,14 @@ app.on('ready', async () => { // eslint-disable-next-line more/no-then appStartInitialSpellcheckSetting = await getSpellCheckSetting(); - await sqlChannels.initialize(); + await sqlChannels.initialize(sql); try { const IDB_KEY = 'indexeddb-delete-needed'; - const item = await sql.getItemById(IDB_KEY); + const item = await sql.sqlCall('getItemById', [IDB_KEY]); if (item && item.value) { - await sql.removeIndexedDBFiles(); - await sql.removeItemById(IDB_KEY); + await sql.sqlCall('removeIndexedDBFiles', []); + await sql.sqlCall('removeItemById', [IDB_KEY]); } } catch (err) { console.log( @@ -1120,16 +1123,18 @@ app.on('ready', async () => { async function cleanupOrphanedAttachments() { const allAttachments = await attachments.getAllAttachments(userDataPath); - const orphanedAttachments = await sql.removeKnownAttachments( - allAttachments - ); + const orphanedAttachments = await sql.sqlCall('removeKnownAttachments', [ + allAttachments, + ]); await attachments.deleteAll({ userDataPath, attachments: orphanedAttachments, }); const allStickers = await attachments.getAllStickers(userDataPath); - const orphanedStickers = await sql.removeKnownStickers(allStickers); + const orphanedStickers = await sql.sqlCall('removeKnownStickers', [ + allStickers, + ]); await attachments.deleteAllStickers({ userDataPath, stickers: orphanedStickers, @@ -1138,8 +1143,9 @@ app.on('ready', async () => { const allDraftAttachments = await attachments.getAllDraftAttachments( userDataPath ); - const orphanedDraftAttachments = await sql.removeKnownDraftAttachments( - allDraftAttachments + const orphanedDraftAttachments = await sql.sqlCall( + 'removeKnownDraftAttachments', + [allDraftAttachments] ); await attachments.deleteAllDraftAttachments({ userDataPath, diff --git a/package.json b/package.json index 83242caaf3..f68ba25f8c 100644 --- a/package.json +++ b/package.json @@ -64,13 +64,13 @@ "fs-xattr": "0.3.0" }, "dependencies": { - "@journeyapps/sqlcipher": "https://github.com/EvanHahn-signal/node-sqlcipher.git#16916949f0c010f6e6d3d5869b10a0ab813eae75", "@sindresorhus/is": "0.8.0", "@types/pino": "6.3.6", "@types/pino-multi-stream": "5.1.0", "abort-controller": "3.0.0", "array-move": "2.1.0", "backbone": "1.3.3", + "better-sqlite3": "https://github.com/indutny/better-sqlite3#a78376d86b5856c14ab4e2f3995f41e1f80df846", "blob-util": "1.3.0", "blueimp-canvas-to-blob": "3.14.0", "blueimp-load-image": "5.14.0", @@ -171,6 +171,7 @@ "@storybook/addons": "5.1.11", "@storybook/react": "5.1.11", "@types/backbone": "1.4.3", + "@types/better-sqlite3": "5.4.1", "@types/blueimp-load-image": "5.14.1", "@types/chai": "4.1.2", "@types/classnames": "2.2.3", @@ -364,6 +365,20 @@ "sgnl" ] }, + "asarUnpack": [ + "js/modules/privacy.js", + "ts/environment.js", + "ts/logging/log.js", + "ts/logging/shared.js", + "ts/sql/Server.js", + "ts/sql/mainWorker.js", + "ts/util/assert.js", + "ts/util/combineNames.js", + "ts/util/enum.js", + "ts/util/isNormalNumber.js", + "ts/util/missingCaseError.js", + "ts/util/reallyJsonStringify.js" + ], "files": [ "package.json", "config/default.json", @@ -405,7 +420,7 @@ "!node_modules/spellchecker/vendor/hunspell/**/*", "!**/node_modules/*/{CHANGELOG.md,README.md,README,readme.md,readme,test,__tests__,tests,powered-test,example,examples,*.d.ts,.snyk-*.flag,benchmark}", "!**/node_modules/.bin", - "!**/node_modules/*/build/**", + "!**/node_modules/**/build/**", "!**/*.{o,hprof,orig,pyc,pyo,rbc}", "!**/._*", "!**/{.DS_Store,.git,.hg,.svn,CVS,RCS,SCCS,__pycache__,thumbs.db,.gitignore,.gitattributes,.flowconfig,.yarn-metadata.json,.idea,appveyor.yml,.travis.yml,circle.yml,npm-debug.log,.nyc_output,yarn.lock,.yarn-integrity}", @@ -413,6 +428,7 @@ "node_modules/websocket/build/Release/*.node", "!node_modules/websocket/builderror.log", "node_modules/ref-napi/build/Release/*.node", + "node_modules/ref-array-napi/node_modules/ref-napi/build/Release/*.node", "node_modules/ffi-napi/build/Release/*.node", "node_modules/socks/build/*.js", "node_modules/socks/build/common/*.js", @@ -420,10 +436,9 @@ "node_modules/smart-buffer/build/*.js", "node_modules/sharp/build/**", "!node_modules/sharp/{install,src,vendor/include,vendor/*/include}", - "!node_modules/@journeyapps/sqlcipher/deps/*", - "!node_modules/@journeyapps/sqlcipher/build/*", - "!node_modules/@journeyapps/sqlcipher/build-tmp-napi-*", - "!node_modules/@journeyapps/sqlcipher/lib/binding/node-*", + "!node_modules/better-sqlite3/deps/*", + "!node_modules/better-sqlite3/src/*", + "node_modules/better-sqlite3/build/Release/*.node", "node_modules/libsignal-client/build/*${platform}*.node", "node_modules/ringrtc/build/${platform}/**", "!**/node_modules/ffi-napi/deps", diff --git a/ts/LibSignalStore.ts b/ts/LibSignalStore.ts index 69355add26..9f28f7d2db 100644 --- a/ts/LibSignalStore.ts +++ b/ts/LibSignalStore.ts @@ -6,6 +6,13 @@ import { fromEncodedBinaryToArrayBuffer, constantTimeEqual } from './Crypto'; import { isNotNil } from './util/isNotNil'; import { isMoreRecentThan } from './util/timestamp'; +import { + IdentityKeyType, + SignedPreKeyType, + PreKeyType, + UnprocessedType, + SessionType, +} from './sql/Interface'; const TIMESTAMP_THRESHOLD = 5 * 1000; // 5 seconds const Direction = { @@ -126,30 +133,6 @@ type KeyPairType = { pubKey: ArrayBuffer; }; -type IdentityKeyType = { - firstUse: boolean; - id: string; - nonblockingApproval: boolean; - publicKey: ArrayBuffer; - timestamp: number; - verified: number; -}; - -type SessionType = { - conversationId: string; - deviceId: number; - id: string; - record: string; -}; - -type SignedPreKeyType = { - confirmed: boolean; - // eslint-disable-next-line camelcase - created_at: number; - id: number; - privateKey: ArrayBuffer; - publicKey: ArrayBuffer; -}; type OuterSignedPrekeyType = { confirmed: boolean; // eslint-disable-next-line camelcase @@ -158,23 +141,6 @@ type OuterSignedPrekeyType = { privKey: ArrayBuffer; pubKey: ArrayBuffer; }; -type PreKeyType = { - id: number; - privateKey: ArrayBuffer; - publicKey: ArrayBuffer; -}; - -type UnprocessedType = { - id: string; - timestamp: number; - version: number; - attempts: number; - envelope: string; - decrypted?: string; - source?: string; - sourceDevice: string; - serverTimestamp: number; -}; // We add a this parameter to avoid an 'implicit any' error on the next line const EventsMixin = (function EventsMixin(this: unknown) { @@ -1175,7 +1141,7 @@ export class SignalProtocolStore extends EventsMixin { return window.Signal.Data.getUnprocessedById(id); } - addUnprocessed(data: UnprocessedType): Promise { + addUnprocessed(data: UnprocessedType): Promise { // We need to pass forceSave because the data has an id already, which will cause // an update instead of an insert. return window.Signal.Data.saveUnprocessed(data, { @@ -1199,7 +1165,9 @@ export class SignalProtocolStore extends EventsMixin { return window.Signal.Data.updateUnprocessedWithData(id, data); } - updateUnprocessedsWithData(items: Array): Promise { + updateUnprocessedsWithData( + items: Array<{ id: string; data: UnprocessedType }> + ): Promise { return window.Signal.Data.updateUnprocessedsWithData(items); } diff --git a/ts/background.ts b/ts/background.ts index c0d4332242..abffaea001 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -3071,6 +3071,10 @@ export async function startApp(): Promise { reconnectTimer = setTimeout(connect, 60000); window.Whisper.events.trigger('reconnectTimer'); + + // If we couldn't connect during startup - we should still switch SQL to + // the main process to avoid stalling UI. + window.sqlInitializer.goBackToMainProcess(); } return; } diff --git a/ts/groups.ts b/ts/groups.ts index cf6126006b..6f973ad51a 100644 --- a/ts/groups.ts +++ b/ts/groups.ts @@ -2319,7 +2319,7 @@ export async function joinGroupV2ViaLinkAndMigrate({ derivedGroupV2Id: undefined, members: undefined, }; - const groupChangeMessages = [ + const groupChangeMessages: Array = [ { ...generateBasicMessage(), type: 'group-v1-migration', @@ -3018,7 +3018,7 @@ async function generateLeftGroupChanges( const isNewlyRemoved = existingMembers.length > (newAttributes.membersV2 || []).length; - const youWereRemovedMessage = { + const youWereRemovedMessage: MessageAttributesType = { ...generateBasicMessage(), type: 'group-v2-change', groupV2Change: { diff --git a/ts/model-types.d.ts b/ts/model-types.d.ts index d14551862a..f91e04a493 100644 --- a/ts/model-types.d.ts +++ b/ts/model-types.d.ts @@ -63,7 +63,7 @@ export type MessageAttributesType = { deletedForEveryoneTimestamp?: number; delivered: number; delivered_to: Array; - errors: Array | null; + errors?: Array; expirationStartTimestamp: number | null; expireTimer: number; expires_at: number; @@ -86,7 +86,7 @@ export type MessageAttributesType = { message: unknown; messageTimer: unknown; profileChange: ProfileNameChangeType; - quote: { + quote?: { attachments: Array; // `author` is an old attribute that holds the author's E164. We shouldn't use it for // new messages, but old messages might have this attribute. @@ -96,8 +96,21 @@ export type MessageAttributesType = { id: string; referencedMessageNotFound: boolean; text: string; - } | null; - reactions: Array<{ fromId: string; emoji: string; timestamp: number }>; + }; + reactions?: Array<{ + emoji: string; + timestamp: number; + fromId: string; + from: { + id: string; + color?: string; + avatarPath?: string; + name?: string; + profileName?: string; + isMe?: boolean; + phoneNumber?: string; + }; + }>; read_by: Array; requiredProtocolVersion: number; sent: boolean; @@ -110,7 +123,19 @@ export type MessageAttributesType = { verifiedChanged: string; id: string; - type?: string; + type?: + | 'incoming' + | 'outgoing' + | 'group' + | 'keychange' + | 'verified-change' + | 'message-history-unsynced' + | 'call-history' + | 'chat-session-refreshed' + | 'group-v1-migration' + | 'group-v2-change' + | 'profile-change' + | 'timer-notification'; body: string; attachments: Array; preview: Array; @@ -135,7 +160,7 @@ export type MessageAttributesType = { flags?: number; groupV2Change?: GroupV2ChangeType; // Required. Used to sort messages in the database for the conversation timeline. - received_at?: number; + received_at: number; received_at_ms?: number; // More of a legacy feature, needed as we were updating the schema of messages in the // background, when we were still in IndexedDB, before attachments had gone to disk @@ -145,7 +170,7 @@ export type MessageAttributesType = { source?: string; sourceUuid?: string; - unread: number; + unread: boolean; timestamp: number; // Backwards-compatibility with prerelease data schema @@ -156,34 +181,37 @@ export type MessageAttributesType = { export type ConversationAttributesTypeType = 'private' | 'group'; export type ConversationAttributesType = { - accessKey: string | null; + accessKey?: string | null; addedBy?: string; capabilities?: CapabilitiesType; color?: string; discoveredUnregisteredAt?: number; - draftAttachments: Array; - draftBodyRanges: Array; - draftTimestamp: number | null; + draftAttachments?: Array<{ + path?: string; + screenshotPath?: string; + }>; + draftBodyRanges?: Array; + draftTimestamp?: number | null; inbox_position: number; isPinned: boolean; lastMessageDeletedForEveryone: boolean; - lastMessageStatus: LastMessageStatus | null; + lastMessageStatus?: LastMessageStatus | null; markedUnread: boolean; messageCount: number; - messageCountBeforeMessageRequests: number | null; - messageRequestResponseType: number; - muteExpiresAt: number | undefined; - profileAvatar: WhatIsThis; - profileKeyCredential: string | null; - profileKeyVersion: string | null; - quotedMessageId: string | null; - sealedSender: unknown; + messageCountBeforeMessageRequests?: number | null; + messageRequestResponseType?: number; + muteExpiresAt?: number; + profileAvatar?: WhatIsThis; + profileKeyCredential?: string | null; + profileKeyVersion?: string | null; + quotedMessageId?: string | null; + sealedSender?: unknown; sentMessageCount: number; - sharedGroupNames: Array; + sharedGroupNames?: Array; id: string; type: ConversationAttributesTypeType; - timestamp: number | null; + timestamp?: number | null; // Shared fields active_at?: number | null; @@ -217,7 +245,7 @@ export type ConversationAttributesType = { // A shorthand, representing whether the user is part of the group. Not strictly for // when the user manually left the group. But historically, that was the only way // to leave a group. - left: boolean; + left?: boolean; groupVersion?: number; // GroupV1 only diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index b91701a126..8820810de9 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -1388,7 +1388,7 @@ export class ConversationModel extends window.Backbone.Model< profileSharing: this.get('profileSharing'), publicParams: this.get('publicParams'), secretParams: this.get('secretParams'), - sharedGroupNames: this.get('sharedGroupNames')!, + sharedGroupNames: this.get('sharedGroupNames'), shouldShowDraft, sortedGroupMembers, timestamp, @@ -2574,7 +2574,7 @@ export class ConversationModel extends window.Backbone.Model< sent_at: now, received_at: window.Signal.Util.incrementMessageCounter(), received_at_ms: now, - unread: true, + unread: 1, changedId: conversationId || this.id, profileChange, // TODO: DESKTOP-722 diff --git a/ts/models/messages.ts b/ts/models/messages.ts index f7234f3860..895350d8a8 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -1745,7 +1745,7 @@ export class MessageModel extends window.Backbone.Model { body: '', bodyRanges: undefined, attachments: [], - quote: null, + quote: undefined, contact: [], sticker: null, preview: [], @@ -2034,7 +2034,7 @@ export class MessageModel extends window.Backbone.Model { return null; } - this.set({ errors: null }); + this.set({ errors: undefined }); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const conversation = this.getConversation()!; @@ -3934,7 +3934,7 @@ export class MessageModel extends window.Backbone.Model { this.clearNotifications(reaction.get('fromId')); } - const newCount = this.get('reactions').length; + const newCount = (this.get('reactions') || []).length; window.log.info( `Done processing reaction for message ${messageId}. Went from ${count} to ${newCount} reactions.` ); diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index e145a4cd93..4957934871 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -36,6 +36,7 @@ import { import { AttachmentDownloadJobType, ClientInterface, + ClientSearchResultMessageType, ClientJobType, ConversationType, IdentityKeyType, @@ -55,7 +56,6 @@ import { import Server from './Server'; import { MessageModel } from '../models/messages'; import { ConversationModel } from '../models/conversations'; -import { waitForPendingQueries } from './Queueing'; // We listen to a lot of events on ipcRenderer, often on the same channel. This prevents // any warnings that might be sent to the console in that case. @@ -243,12 +243,14 @@ const dataInterface: ClientInterface = { export default dataInterface; async function goBackToMainProcess(): Promise { - window.log.info('data.goBackToMainProcess: waiting for pending queries'); - - // Let pending queries finish before we'll give write access to main process. - // We don't want to be writing from two processes at the same time! - await waitForPendingQueries(); + if (!shouldUseRendererProcess) { + window.log.info( + 'data.goBackToMainProcess: already switched to main process' + ); + return; + } + // We don't need to wait for pending queries since they are synchronous. window.log.info('data.goBackToMainProcess: switching to main process'); shouldUseRendererProcess = false; @@ -514,8 +516,6 @@ function keysFromArrayBuffer(keys: Array, data: any) { // Top-level calls async function shutdown() { - await waitForPendingQueries(); - // Stop accepting new SQL jobs, flush outstanding queue await _shutdown(); @@ -761,7 +761,13 @@ const updateConversationBatcher = createBatcher({ // We only care about the most recent update for each conversation const byId = groupBy(items, item => item.id); const ids = Object.keys(byId); - const mostRecent = ids.map(id => last(byId[id])); + const mostRecent = ids.map( + (id: string): ConversationType => { + const maybeLast = last(byId[id]); + assert(maybeLast !== undefined, 'Empty array in `groupBy` result'); + return maybeLast; + } + ); await updateConversations(mostRecent); }, @@ -857,9 +863,13 @@ async function searchConversations(query: string) { return conversations; } -function handleSearchMessageJSON(messages: Array) { +function handleSearchMessageJSON( + messages: Array +): Array { return messages.map(message => ({ + json: message.json, ...JSON.parse(message.json), + bodyRanges: [], snippet: message.snippet, })); } @@ -940,7 +950,7 @@ async function getMessageById( ) { const message = await channels.getMessageById(id); if (!message) { - return null; + return undefined; } return new Message(message); @@ -1262,7 +1272,9 @@ async function updateUnprocessedAttempts(id: string, attempts: number) { async function updateUnprocessedWithData(id: string, data: UnprocessedType) { await channels.updateUnprocessedWithData(id, data); } -async function updateUnprocessedsWithData(array: Array) { +async function updateUnprocessedsWithData( + array: Array<{ id: string; data: UnprocessedType }> +) { await channels.updateUnprocessedsWithData(array); } diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 4921a17192..e45c4ce3b0 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -4,31 +4,129 @@ /* eslint-disable @typescript-eslint/ban-types */ /* eslint-disable camelcase */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { LocaleMessagesType } from '../types/I18N'; import { + ConversationAttributesType, ConversationModelCollectionType, + MessageAttributesType, MessageModelCollectionType, } from '../model-types.d'; import { MessageModel } from '../models/messages'; import { ConversationModel } from '../models/conversations'; -export type AttachmentDownloadJobType = any; -export type ConverationMetricsType = any; -export type ConversationType = any; -export type EmojiType = any; -export type IdentityKeyType = any; +export type AttachmentDownloadJobType = { + id: string; + timestamp: number; + pending: number; + attempts: number; +}; +export type MessageMetricsType = { + id: string; + // eslint-disable-next-line camelcase + received_at: number; + // eslint-disable-next-line camelcase + sent_at: number; +}; +export type ConversationMetricsType = { + oldest?: MessageMetricsType; + newest?: MessageMetricsType; + oldestUnread?: MessageMetricsType; + totalUnread: number; +}; +export type ConversationType = ConversationAttributesType; +export type EmojiType = { + shortName: string; + lastUsage: number; +}; +export type IdentityKeyType = { + firstUse: boolean; + id: string; + nonblockingApproval: boolean; + publicKey: ArrayBuffer; + timestamp: number; + verified: number; +}; export type ItemType = any; -export type MessageType = any; -export type MessageTypeUnhydrated = any; -export type PreKeyType = any; -export type SearchResultMessageType = any; -export type SessionType = any; -export type SignedPreKeyType = any; -export type StickerPackStatusType = string; -export type StickerPackType = any; -export type StickerType = any; -export type UnprocessedType = any; +export type MessageType = MessageAttributesType; +export type MessageTypeUnhydrated = { + json: string; +}; +export type PreKeyType = { + id: number; + privateKey: ArrayBuffer; + publicKey: ArrayBuffer; +}; +export type SearchResultMessageType = { + json: string; + snippet: string; +}; +export type ClientSearchResultMessageType = MessageType & { + json: string; + bodyRanges: []; + snippet: string; +}; +export type SessionType = { + id: string; + conversationId: string; + deviceId: number; + record: string; +}; +export type SignedPreKeyType = { + confirmed: boolean; + // eslint-disable-next-line camelcase + created_at: number; + id: number; + privateKey: ArrayBuffer; + publicKey: ArrayBuffer; +}; +export type StickerPackStatusType = + | 'known' + | 'ephemeral' + | 'downloaded' + | 'installed' + | 'pending' + | 'error'; + +export type StickerType = { + id: number; + packId: string; + + emoji: string; + isCoverOnly: string; + lastUsed: number; + path: string; + width: number; + height: number; +}; +export type StickerPackType = { + id: string; + key: string; + + attemptedStatus: 'downloaded' | 'installed' | 'ephemeral'; + author: string; + coverStickerId: number; + createdAt: number; + downloadAttempts: number; + installedAt: number | null; + lastUsed: number; + status: StickerPackStatusType; + stickerCount: number; + stickers: ReadonlyArray; + title: string; +}; +export type UnprocessedType = { + id: string; + timestamp: number; + version: number; + attempts: number; + envelope: string; + + source?: string; + sourceUuid?: string; + sourceDevice?: string; + serverTimestamp?: number; + decrypted?: string; +}; export type DataInterface = { close: () => Promise; @@ -84,15 +182,6 @@ export type DataInterface = { query: string, options?: { limit?: number } ) => Promise>; - searchMessages: ( - query: string, - options?: { limit?: number } - ) => Promise>; - searchMessagesInConversation: ( - query: string, - conversationId: string, - options?: { limit?: number } - ) => Promise>; getMessageCount: (conversationId?: string) => Promise; saveMessages: ( @@ -102,7 +191,7 @@ export type DataInterface = { getAllMessageIds: () => Promise>; getMessageMetricsForConversation: ( conversationId: string - ) => Promise; + ) => Promise; hasGroupCallHistoryMessage: ( conversationId: string, eraId: string @@ -117,13 +206,15 @@ export type DataInterface = { saveUnprocessed: ( data: UnprocessedType, options?: { forceSave?: boolean } - ) => Promise; + ) => Promise; updateUnprocessedAttempts: (id: string, attempts: number) => Promise; updateUnprocessedWithData: ( id: string, data: UnprocessedType ) => Promise; - updateUnprocessedsWithData: (array: Array) => Promise; + updateUnprocessedsWithData: ( + array: Array<{ id: string; data: UnprocessedType }> + ) => Promise; getUnprocessedById: (id: string) => Promise; saveUnprocesseds: ( arrayOfUnprocessed: Array, @@ -203,7 +294,7 @@ export type ServerInterface = DataInterface & { getAllConversations: () => Promise>; getAllGroupsInvolvingId: (id: string) => Promise>; getAllPrivateConversations: () => Promise>; - getConversationById: (id: string) => Promise; + getConversationById: (id: string) => Promise; getExpiredMessages: () => Promise>; getMessageById: (id: string) => Promise; getMessageBySender: (options: { @@ -234,8 +325,8 @@ export type ServerInterface = DataInterface & { conversationId: string; ourConversationId: string; }) => Promise; - getNextExpiringMessage: () => Promise; - getNextTapToViewMessageToAgeOut: () => Promise; + getNextExpiringMessage: () => Promise; + getNextTapToViewMessageToAgeOut: () => Promise; getOutgoingWithoutExpiresAt: () => Promise>; getTapToViewMessagesNeedingErase: () => Promise>; getUnreadByConversation: ( @@ -244,6 +335,15 @@ export type ServerInterface = DataInterface & { removeConversation: (id: Array | string) => Promise; removeMessage: (id: string) => Promise; removeMessages: (ids: Array) => Promise; + searchMessages: ( + query: string, + options?: { limit?: number } + ) => Promise>; + searchMessagesInConversation: ( + query: string, + conversationId: string, + options?: { limit?: number } + ) => Promise>; saveMessage: ( data: MessageType, options: { forceSave?: boolean } @@ -255,11 +355,7 @@ export type ServerInterface = DataInterface & { // Server-only - initialize: (options: { - configDir: string; - key: string; - messages: LocaleMessagesType; - }) => Promise; + initialize: (options: { configDir: string; key: string }) => Promise; initializeRenderer: (options: { configDir: string; @@ -298,7 +394,7 @@ export type ClientInterface = DataInterface & { getMessageById: ( id: string, options: { Message: typeof MessageModel } - ) => Promise; + ) => Promise; getMessageBySender: ( data: { source: string; @@ -373,6 +469,15 @@ export type ClientInterface = DataInterface & { data: MessageType, options: { forceSave?: boolean; Message: typeof MessageModel } ) => Promise; + searchMessages: ( + query: string, + options?: { limit?: number } + ) => Promise>; + searchMessagesInConversation: ( + query: string, + conversationId: string, + options?: { limit?: number } + ) => Promise>; updateConversation: (data: ConversationType, extra?: unknown) => void; // Test-only diff --git a/ts/sql/Queueing.ts b/ts/sql/Queueing.ts deleted file mode 100644 index 43659edf5a..0000000000 --- a/ts/sql/Queueing.ts +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2018-2020 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only - -import Queue from 'p-queue'; -import { ServerInterface } from './Interface'; - -let allQueriesDone: () => void | undefined; -let sqlQueries = 0; -let singleQueue: Queue | null = null; -let multipleQueue: Queue | null = null; - -// Note: we don't want queue timeouts, because delays here are due to in-progress sql -// operations. For example we might try to start a transaction when the previous isn't -// done, causing that database operation to fail. -function makeNewSingleQueue(): Queue { - singleQueue = new Queue({ concurrency: 1 }); - return singleQueue; -} -function makeNewMultipleQueue(): Queue { - multipleQueue = new Queue({ concurrency: 10 }); - return multipleQueue; -} - -const DEBUG = false; - -function makeSQLJob( - fn: ServerInterface[keyof ServerInterface], - args: Array, - callName: keyof ServerInterface -) { - if (DEBUG) { - // eslint-disable-next-line no-console - console.log(`SQL(${callName}) queued`); - } - return async () => { - sqlQueries += 1; - const start = Date.now(); - if (DEBUG) { - // eslint-disable-next-line no-console - console.log(`SQL(${callName}) started`); - } - let result; - try { - // Ignoring this error TS2556: Expected 3 arguments, but got 0 or more. - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - result = await fn(...args); - } finally { - sqlQueries -= 1; - if (allQueriesDone && sqlQueries <= 0) { - allQueriesDone(); - } - } - const end = Date.now(); - const delta = end - start; - if (DEBUG || delta > 10) { - // eslint-disable-next-line no-console - console.log(`SQL(${callName}) succeeded in ${end - start}ms`); - } - return result; - }; -} - -async function handleCall( - fn: ServerInterface[keyof ServerInterface], - args: Array, - callName: keyof ServerInterface -) { - if (!fn) { - throw new Error(`sql channel: ${callName} is not an available function`); - } - - let result; - - // We queue here to keep multi-query operations atomic. Without it, any multistage - // data operation (even within a BEGIN/COMMIT) can become interleaved, since all - // requests share one database connection. - - // A needsSerial method must be run in our single concurrency queue. - if (fn.needsSerial) { - if (singleQueue) { - result = await singleQueue.add(makeSQLJob(fn, args, callName)); - } else if (multipleQueue) { - const queue = makeNewSingleQueue(); - - const multipleQueueLocal = multipleQueue; - queue.add(() => multipleQueueLocal.onIdle()); - multipleQueue = null; - - result = await queue.add(makeSQLJob(fn, args, callName)); - } else { - const queue = makeNewSingleQueue(); - result = await queue.add(makeSQLJob(fn, args, callName)); - } - } else { - // The request can be parallelized. To keep the same structure as the above block - // we force this section into the 'lonely if' pattern. - // eslint-disable-next-line no-lonely-if - if (multipleQueue) { - result = await multipleQueue.add(makeSQLJob(fn, args, callName)); - } else if (singleQueue) { - const queue = makeNewMultipleQueue(); - queue.pause(); - - const singleQueueRef = singleQueue; - - singleQueue = null; - const promise = queue.add(makeSQLJob(fn, args, callName)); - if (singleQueueRef) { - await singleQueueRef.onIdle(); - } - - queue.start(); - result = await promise; - } else { - const queue = makeNewMultipleQueue(); - result = await queue.add(makeSQLJob(fn, args, callName)); - } - } - - return result; -} - -export async function waitForPendingQueries(): Promise { - return new Promise(resolve => { - if (sqlQueries === 0) { - resolve(); - } else { - allQueriesDone = () => resolve(); - } - }); -} - -export function applyQueueing(dataInterface: ServerInterface): ServerInterface { - return Object.keys(dataInterface).reduce((acc, callName) => { - const serverInterfaceKey = callName as keyof ServerInterface; - acc[serverInterfaceKey] = async (...args: Array) => - handleCall(dataInterface[serverInterfaceKey], args, serverInterfaceKey); - return acc; - }, {} as ServerInterface); -} diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index e27219afc5..f03892d35f 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -4,7 +4,6 @@ /* eslint-disable no-nested-ternary */ /* eslint-disable camelcase */ /* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -/* eslint-disable no-await-in-loop */ /* eslint-disable no-restricted-syntax */ /* eslint-disable no-console */ /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -12,10 +11,8 @@ import { join } from 'path'; import mkdirp from 'mkdirp'; import rimraf from 'rimraf'; -import PQueue from 'p-queue'; -import sql from '@journeyapps/sqlcipher'; +import SQL, { Database } from 'better-sqlite3'; -import pify from 'pify'; import { v4 as generateUUID } from 'uuid'; import { Dictionary, @@ -23,7 +20,6 @@ import { fromPairs, isNil, isNumber, - isObject, isString, keyBy, last, @@ -37,14 +33,17 @@ import { isNormalNumber } from '../util/isNormalNumber'; import { combineNames } from '../util/combineNames'; import { GroupV2MemberType } from '../model-types.d'; -import { LocaleMessagesType } from '../types/I18N'; import { AttachmentDownloadJobType, + ConversationMetricsType, ConversationType, + EmojiType, IdentityKeyType, ItemType, MessageType, + MessageTypeUnhydrated, + MessageMetricsType, PreKeyType, SearchResultMessageType, ServerInterface, @@ -55,7 +54,6 @@ import { StickerType, UnprocessedType, } from './Interface'; -import { applyQueueing } from './Queueing'; declare global { // We want to extend `Function`'s properties, so we need to use an interface. @@ -65,6 +63,17 @@ declare global { } } +type JSONRows = Array<{ readonly json: string }>; +type ConversationRow = Readonly<{ + json: string; + profileLastFetchedAt: null | number; +}>; +type ConversationRows = Array; + +type EmptyQuery = []; +type ArrayQuery = Array>; +type Query = { [key: string]: null | number | string }; + // Because we can't force this module to conform to an interface, we narrow our exports // to this one default export, which does conform to the interface. // Note: In Javascript, you need to access the .default property when requiring it @@ -203,8 +212,7 @@ const dataInterface: ServerInterface = { removeKnownStickers, removeKnownDraftAttachments, }; - -export default applyQueueing(dataInterface); +export default dataInterface; function objectToJSON(data: any) { return JSON.stringify(data); @@ -212,12 +220,7 @@ function objectToJSON(data: any) { function jsonToObject(json: string): any { return JSON.parse(json); } -function rowToConversation( - row: Readonly<{ - json: string; - profileLastFetchedAt: null | number; - }> -): ConversationType { +function rowToConversation(row: ConversationRow): ConversationType { const parsedJson = JSON.parse(row.json); let profileLastFetchedAt: undefined | number; @@ -245,557 +248,405 @@ function isRenderer() { return process.type === 'renderer'; } -async function openDatabase(filePath: string): Promise { - return new Promise((resolve, reject) => { - let instance: sql.Database | undefined; - const callback = (error: Error | null) => { - if (error) { - reject(error); +function getSQLiteVersion(db: Database): string { + const { sqlite_version } = db + .prepare('select sqlite_version() AS sqlite_version') + .get(); - return; - } - if (!instance) { - reject(new Error('openDatabase: Unable to get database instance')); - - return; - } - - resolve(instance); - }; - - instance = new sql.Database(filePath, callback); - - // See: https://github.com/mapbox/node-sqlite3/issues/1395 - instance.serialize(); - }); + return sqlite_version; } -type PromisifiedSQLDatabase = { - close: () => Promise; - run: (statement: string, params?: { [key: string]: any }) => Promise; - get: (statement: string, params?: { [key: string]: any }) => Promise; - all: ( - statement: string, - params?: { [key: string]: any } - ) => Promise>; - on: (event: 'trace', handler: (sql: string) => void) => void; -}; - -function promisify(rawInstance: sql.Database): PromisifiedSQLDatabase { - return { - close: pify(rawInstance.close.bind(rawInstance)), - run: pify(rawInstance.run.bind(rawInstance)), - get: pify(rawInstance.get.bind(rawInstance)), - all: pify(rawInstance.all.bind(rawInstance)), - on: rawInstance.on.bind(rawInstance), - }; +function getSchemaVersion(db: Database): number { + return db.pragma('schema_version', { simple: true }); } -async function getSQLiteVersion(instance: PromisifiedSQLDatabase) { - const row = await instance.get('select sqlite_version() AS sqlite_version'); - - return row.sqlite_version; -} - -async function getSchemaVersion(instance: PromisifiedSQLDatabase) { - const row = await instance.get('PRAGMA schema_version;'); - - return row.schema_version; -} - -async function setUserVersion( - instance: PromisifiedSQLDatabase, - version: number -) { +function setUserVersion(db: Database, version: number): void { if (!isNumber(version)) { throw new Error(`setUserVersion: version ${version} is not a number`); } - await instance.get(`PRAGMA user_version = ${version};`); + db.pragma(`user_version = ${version}`); } -async function keyDatabase(instance: PromisifiedSQLDatabase, key: string) { +function keyDatabase(db: Database, key: string): void { // https://www.zetetic.net/sqlcipher/sqlcipher-api/#key - await instance.run(`PRAGMA key = "x'${key}'";`); + db.pragma(`key = "x'${key}'"`); // https://sqlite.org/wal.html - await instance.run('PRAGMA journal_mode = WAL;'); - await instance.run('PRAGMA synchronous = NORMAL;'); + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); } -async function getUserVersion(instance: PromisifiedSQLDatabase) { - const row = await instance.get('PRAGMA user_version;'); - - return row.user_version; +function getUserVersion(db: Database): number { + return db.pragma('user_version', { simple: true }); } -async function getSQLCipherVersion(instance: PromisifiedSQLDatabase) { - const row = await instance.get('PRAGMA cipher_version;'); - try { - return row.cipher_version; - } catch (e) { - return null; +function getSQLCipherVersion(db: Database): string | undefined { + return db.pragma('cipher_version', { simple: true }); +} + +function getSQLCipherIntegrityCheck(db: Database): Array | undefined { + const rows: Array<{ cipher_integrity_check: string }> = db.pragma( + 'cipher_integrity_check' + ); + if (rows.length === 0) { + return undefined; } + return rows.map(row => row.cipher_integrity_check); } -async function getSQLCipherIntegrityCheck(instance: PromisifiedSQLDatabase) { - const row = await instance.get('PRAGMA cipher_integrity_check;'); - if (row) { - return row.cipher_integrity_check; +function getSQLIntegrityCheck(db: Database): string | undefined { + const checkResult = db.pragma('integrity_check', { simple: true }); + if (checkResult !== 'ok') { + return checkResult; } - return null; + return undefined; } -async function getSQLIntegrityCheck(instance: PromisifiedSQLDatabase) { - const row = await instance.get('PRAGMA integrity_check;'); - if (row && row.integrity_check !== 'ok') { - return row.integrity_check; - } - - return null; -} - -async function migrateSchemaVersion(instance: PromisifiedSQLDatabase) { - const userVersion = await getUserVersion(instance); +function migrateSchemaVersion(db: Database): void { + const userVersion = getUserVersion(db); if (userVersion > 0) { return; } - const schemaVersion = await getSchemaVersion(instance); + const schemaVersion = getSchemaVersion(db); const newUserVersion = schemaVersion > 18 ? 16 : schemaVersion; console.log( - `migrateSchemaVersion: Migrating from schema_version ${schemaVersion} to user_version ${newUserVersion}` + 'migrateSchemaVersion: Migrating from schema_version ' + + `${schemaVersion} to user_version ${newUserVersion}` ); - await setUserVersion(instance, newUserVersion); + setUserVersion(db, newUserVersion); } -async function openAndMigrateDatabase(filePath: string, key: string) { - let promisified: PromisifiedSQLDatabase | undefined; +function openAndMigrateDatabase(filePath: string, key: string) { + let db: Database | undefined; // First, we try to open the database without any cipher changes try { - const instance = await openDatabase(filePath); - promisified = promisify(instance); - await keyDatabase(promisified, key); + db = new SQL(filePath); + keyDatabase(db, key); + migrateSchemaVersion(db); - await migrateSchemaVersion(promisified); - - return promisified; + return db; } catch (error) { - if (promisified) { - await promisified.close(); + if (db) { + db.close(); } console.log('migrateDatabase: Migration without cipher change failed'); } // If that fails, we try to open the database with 3.x compatibility to extract the // user_version (previously stored in schema_version, blown away by cipher_migrate). - const instance1 = await openDatabase(filePath); - promisified = promisify(instance1); - await keyDatabase(promisified, key); + db = new SQL(filePath); + keyDatabase(db, key); // https://www.zetetic.net/blog/2018/11/30/sqlcipher-400-release/#compatability-sqlcipher-4-0-0 - await promisified.run('PRAGMA cipher_compatibility = 3;'); - await migrateSchemaVersion(promisified); - await promisified.close(); + db.pragma('cipher_compatibility = 3'); + migrateSchemaVersion(db); + db.close(); // After migrating user_version -> schema_version, we reopen database, because we can't // migrate to the latest ciphers after we've modified the defaults. - const instance2 = await openDatabase(filePath); - promisified = promisify(instance2); - await keyDatabase(promisified, key); + db = new SQL(filePath); + keyDatabase(db, key); - await promisified.run('PRAGMA cipher_migrate;'); + db.pragma('cipher_migrate'); - return promisified; + return db; } const INVALID_KEY = /[^0-9A-Fa-f]/; -async function openAndSetUpSQLCipher( - filePath: string, - { key }: { key: string } -) { +function openAndSetUpSQLCipher(filePath: string, { key }: { key: string }) { const match = INVALID_KEY.exec(key); if (match) { throw new Error(`setupSQLCipher: key '${key}' is not valid`); } - const instance = await openAndMigrateDatabase(filePath, key); + const db = openAndMigrateDatabase(filePath, key); // Because foreign key support is not enabled by default! - await instance.run('PRAGMA foreign_keys = ON;'); + db.pragma('foreign_keys = ON'); - return instance; + return db; } -async function updateToSchemaVersion1( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion1(currentVersion: number, db: Database): void { if (currentVersion >= 1) { return; } console.log('updateToSchemaVersion1: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + CREATE TABLE messages( + id STRING PRIMARY KEY ASC, + json TEXT, - try { - await instance.run( - `CREATE TABLE messages( - id STRING PRIMARY KEY ASC, - json TEXT, + unread INTEGER, + expires_at INTEGER, + sent_at INTEGER, + schemaVersion INTEGER, + conversationId STRING, + received_at INTEGER, + source STRING, + sourceDevice STRING, + hasAttachments INTEGER, + hasFileAttachments INTEGER, + hasVisualMediaAttachments INTEGER + ); + CREATE INDEX messages_unread ON messages ( + unread + ); + CREATE INDEX messages_expires_at ON messages ( + expires_at + ); + CREATE INDEX messages_receipt ON messages ( + sent_at + ); + CREATE INDEX messages_schemaVersion ON messages ( + schemaVersion + ); + CREATE INDEX messages_conversation ON messages ( + conversationId, + received_at + ); + CREATE INDEX messages_duplicate_check ON messages ( + source, + sourceDevice, + sent_at + ); + CREATE INDEX messages_hasAttachments ON messages ( + conversationId, + hasAttachments, + received_at + ); + CREATE INDEX messages_hasFileAttachments ON messages ( + conversationId, + hasFileAttachments, + received_at + ); + CREATE INDEX messages_hasVisualMediaAttachments ON messages ( + conversationId, + hasVisualMediaAttachments, + received_at + ); + CREATE TABLE unprocessed( + id STRING, + timestamp INTEGER, + json TEXT + ); + CREATE INDEX unprocessed_id ON unprocessed ( + id + ); + CREATE INDEX unprocessed_timestamp ON unprocessed ( + timestamp + ); + `); - unread INTEGER, - expires_at INTEGER, - sent_at INTEGER, - schemaVersion INTEGER, - conversationId STRING, - received_at INTEGER, - source STRING, - sourceDevice STRING, - hasAttachments INTEGER, - hasFileAttachments INTEGER, - hasVisualMediaAttachments INTEGER - );` - ); + db.pragma('user_version = 1'); + })(); - await instance.run(`CREATE INDEX messages_unread ON messages ( - unread - );`); - await instance.run(`CREATE INDEX messages_expires_at ON messages ( - expires_at - );`); - await instance.run(`CREATE INDEX messages_receipt ON messages ( - sent_at - );`); - await instance.run(`CREATE INDEX messages_schemaVersion ON messages ( - schemaVersion - );`); - - await instance.run(`CREATE INDEX messages_conversation ON messages ( - conversationId, - received_at - );`); - - await instance.run(`CREATE INDEX messages_duplicate_check ON messages ( - source, - sourceDevice, - sent_at - );`); - await instance.run(`CREATE INDEX messages_hasAttachments ON messages ( - conversationId, - hasAttachments, - received_at - );`); - await instance.run(`CREATE INDEX messages_hasFileAttachments ON messages ( - conversationId, - hasFileAttachments, - received_at - );`); - await instance.run(`CREATE INDEX messages_hasVisualMediaAttachments ON messages ( - conversationId, - hasVisualMediaAttachments, - received_at - );`); - - await instance.run(`CREATE TABLE unprocessed( - id STRING, - timestamp INTEGER, - json TEXT - );`); - await instance.run(`CREATE INDEX unprocessed_id ON unprocessed ( - id - );`); - await instance.run(`CREATE INDEX unprocessed_timestamp ON unprocessed ( - timestamp - );`); - - await instance.run('PRAGMA user_version = 1;'); - await instance.run('COMMIT TRANSACTION;'); - - console.log('updateToSchemaVersion1: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + console.log('updateToSchemaVersion1: success!'); } -async function updateToSchemaVersion2( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion2(currentVersion: number, db: Database): void { if (currentVersion >= 2) { return; } console.log('updateToSchemaVersion2: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + ALTER TABLE messages + ADD COLUMN expireTimer INTEGER; - try { - await instance.run( - `ALTER TABLE messages - ADD COLUMN expireTimer INTEGER;` - ); + ALTER TABLE messages + ADD COLUMN expirationStartTimestamp INTEGER; - await instance.run( - `ALTER TABLE messages - ADD COLUMN expirationStartTimestamp INTEGER;` - ); + ALTER TABLE messages + ADD COLUMN type STRING; - await instance.run( - `ALTER TABLE messages - ADD COLUMN type STRING;` - ); + CREATE INDEX messages_expiring ON messages ( + expireTimer, + expirationStartTimestamp, + expires_at + ); - await instance.run(`CREATE INDEX messages_expiring ON messages ( - expireTimer, - expirationStartTimestamp, - expires_at - );`); - - await instance.run( - `UPDATE messages SET - expirationStartTimestamp = json_extract(json, '$.expirationStartTimestamp'), - expireTimer = json_extract(json, '$.expireTimer'), - type = json_extract(json, '$.type');` - ); - - await instance.run('PRAGMA user_version = 2;'); - await instance.run('COMMIT TRANSACTION;'); - - console.log('updateToSchemaVersion2: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + UPDATE messages SET + expirationStartTimestamp = json_extract(json, '$.expirationStartTimestamp'), + expireTimer = json_extract(json, '$.expireTimer'), + type = json_extract(json, '$.type'); + `); + db.pragma('user_version = 2'); + })(); + console.log('updateToSchemaVersion2: success!'); } -async function updateToSchemaVersion3( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion3(currentVersion: number, db: Database): void { if (currentVersion >= 3) { return; } console.log('updateToSchemaVersion3: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + DROP INDEX messages_expiring; + DROP INDEX messages_unread; - try { - await instance.run('DROP INDEX messages_expiring;'); - await instance.run('DROP INDEX messages_unread;'); + CREATE INDEX messages_without_timer ON messages ( + expireTimer, + expires_at, + type + ) WHERE expires_at IS NULL AND expireTimer IS NOT NULL; - await instance.run(`CREATE INDEX messages_without_timer ON messages ( - expireTimer, - expires_at, - type - ) WHERE expires_at IS NULL AND expireTimer IS NOT NULL;`); + CREATE INDEX messages_unread ON messages ( + conversationId, + unread + ) WHERE unread IS NOT NULL; - await instance.run(`CREATE INDEX messages_unread ON messages ( - conversationId, - unread - ) WHERE unread IS NOT NULL;`); + ANALYZE; + `); - await instance.run('ANALYZE;'); - await instance.run('PRAGMA user_version = 3;'); - await instance.run('COMMIT TRANSACTION;'); + db.pragma('user_version = 3'); + })(); - console.log('updateToSchemaVersion3: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + console.log('updateToSchemaVersion3: success!'); } -async function updateToSchemaVersion4( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion4(currentVersion: number, db: Database): void { if (currentVersion >= 4) { return; } console.log('updateToSchemaVersion4: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + CREATE TABLE conversations( + id STRING PRIMARY KEY ASC, + json TEXT, - try { - await instance.run( - `CREATE TABLE conversations( - id STRING PRIMARY KEY ASC, - json TEXT, + active_at INTEGER, + type STRING, + members TEXT, + name TEXT, + profileName TEXT + ); + CREATE INDEX conversations_active ON conversations ( + active_at + ) WHERE active_at IS NOT NULL; - active_at INTEGER, - type STRING, - members TEXT, - name TEXT, - profileName TEXT - );` - ); + CREATE INDEX conversations_type ON conversations ( + type + ) WHERE type IS NOT NULL; + `); - await instance.run(`CREATE INDEX conversations_active ON conversations ( - active_at - ) WHERE active_at IS NOT NULL;`); + db.pragma('user_version = 4'); + })(); - await instance.run(`CREATE INDEX conversations_type ON conversations ( - type - ) WHERE type IS NOT NULL;`); - - await instance.run('PRAGMA user_version = 4;'); - await instance.run('COMMIT TRANSACTION;'); - - console.log('updateToSchemaVersion4: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + console.log('updateToSchemaVersion4: success!'); } -async function updateToSchemaVersion6( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion6(currentVersion: number, db: Database): void { if (currentVersion >= 6) { return; } console.log('updateToSchemaVersion6: starting...'); - await instance.run('BEGIN TRANSACTION;'); - try { - // key-value, ids are strings, one extra column - await instance.run( - `CREATE TABLE sessions( - id STRING PRIMARY KEY ASC, - number STRING, - json TEXT - );` - ); + db.transaction(() => { + db.exec(` + -- key-value, ids are strings, one extra column + CREATE TABLE sessions( + id STRING PRIMARY KEY ASC, + number STRING, + json TEXT + ); + CREATE INDEX sessions_number ON sessions ( + number + ) WHERE number IS NOT NULL; + -- key-value, ids are strings + CREATE TABLE groups( + id STRING PRIMARY KEY ASC, + json TEXT + ); + CREATE TABLE identityKeys( + id STRING PRIMARY KEY ASC, + json TEXT + ); + CREATE TABLE items( + id STRING PRIMARY KEY ASC, + json TEXT + ); + -- key-value, ids are integers + CREATE TABLE preKeys( + id INTEGER PRIMARY KEY ASC, + json TEXT + ); + CREATE TABLE signedPreKeys( + id INTEGER PRIMARY KEY ASC, + json TEXT + ); + `); - await instance.run(`CREATE INDEX sessions_number ON sessions ( - number - ) WHERE number IS NOT NULL;`); + db.pragma('user_version = 6'); + })(); - // key-value, ids are strings - await instance.run( - `CREATE TABLE groups( - id STRING PRIMARY KEY ASC, - json TEXT - );` - ); - await instance.run( - `CREATE TABLE identityKeys( - id STRING PRIMARY KEY ASC, - json TEXT - );` - ); - await instance.run( - `CREATE TABLE items( - id STRING PRIMARY KEY ASC, - json TEXT - );` - ); - - // key-value, ids are integers - await instance.run( - `CREATE TABLE preKeys( - id INTEGER PRIMARY KEY ASC, - json TEXT - );` - ); - await instance.run( - `CREATE TABLE signedPreKeys( - id INTEGER PRIMARY KEY ASC, - json TEXT - );` - ); - - await instance.run('PRAGMA user_version = 6;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion6: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + console.log('updateToSchemaVersion6: success!'); } -async function updateToSchemaVersion7( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion7(currentVersion: number, db: Database): void { if (currentVersion >= 7) { return; } console.log('updateToSchemaVersion7: starting...'); - await instance.run('BEGIN TRANSACTION;'); - try { - // SQLite has been coercing our STRINGs into numbers, so we force it with TEXT - // We create a new table then copy the data into it, since we can't modify columns + db.transaction(() => { + db.exec(` + -- SQLite has been coercing our STRINGs into numbers, so we force it with TEXT + -- We create a new table then copy the data into it, since we can't modify columns + DROP INDEX sessions_number; + ALTER TABLE sessions RENAME TO sessions_old; - await instance.run('DROP INDEX sessions_number;'); - await instance.run('ALTER TABLE sessions RENAME TO sessions_old;'); - - await instance.run( - `CREATE TABLE sessions( + CREATE TABLE sessions( id TEXT PRIMARY KEY, number TEXT, json TEXT - );` - ); - - await instance.run(`CREATE INDEX sessions_number ON sessions ( - number - ) WHERE number IS NOT NULL;`); - - await instance.run(`INSERT INTO sessions(id, number, json) - SELECT "+" || id, number, json FROM sessions_old; + ); + CREATE INDEX sessions_number ON sessions ( + number + ) WHERE number IS NOT NULL; + INSERT INTO sessions(id, number, json) + SELECT "+" || id, number, json FROM sessions_old; + DROP TABLE sessions_old; `); - await instance.run('DROP TABLE sessions_old;'); - - await instance.run('PRAGMA user_version = 7;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion7: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 7'); + })(); + console.log('updateToSchemaVersion7: success!'); } -async function updateToSchemaVersion8( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion8(currentVersion: number, db: Database): void { if (currentVersion >= 8) { return; } console.log('updateToSchemaVersion8: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + -- First, we pull a new body field out of the message table's json blob + ALTER TABLE messages + ADD COLUMN body TEXT; + UPDATE messages SET body = json_extract(json, '$.body'); - try { - // First, we pull a new body field out of the message table's json blob - await instance.run( - `ALTER TABLE messages - ADD COLUMN body TEXT;` - ); - await instance.run( - "UPDATE messages SET body = json_extract(json, '$.body')" - ); - - // Then we create our full-text search table and populate it - await instance.run(` + -- Then we create our full-text search table and populate it CREATE VIRTUAL TABLE messages_fts - USING fts5(id UNINDEXED, body); - `); - await instance.run(` - INSERT INTO messages_fts(id, body) - SELECT id, body FROM messages; - `); + USING fts5(id UNINDEXED, body); - // Then we set up triggers to keep the full-text search table up to date - await instance.run(` + INSERT INTO messages_fts(id, body) + SELECT id, body FROM messages; + + -- Then we set up triggers to keep the full-text search table up to date CREATE TRIGGER messages_on_insert AFTER INSERT ON messages BEGIN INSERT INTO messages_fts ( id, @@ -805,13 +656,9 @@ async function updateToSchemaVersion8( new.body ); END; - `); - await instance.run(` CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN DELETE FROM messages_fts WHERE id = old.id; END; - `); - await instance.run(` CREATE TRIGGER messages_on_update AFTER UPDATE ON messages BEGIN DELETE FROM messages_fts WHERE id = old.id; INSERT INTO messages_fts( @@ -828,351 +675,285 @@ async function updateToSchemaVersion8( // https://sqlite.org/fts5.html#the_highlight_function // https://sqlite.org/fts5.html#the_snippet_function - await instance.run('PRAGMA user_version = 8;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion8: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 8'); + })(); + console.log('updateToSchemaVersion8: success!'); } -async function updateToSchemaVersion9( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion9(currentVersion: number, db: Database): void { if (currentVersion >= 9) { return; } console.log('updateToSchemaVersion9: starting...'); - await instance.run('BEGIN TRANSACTION;'); - try { - await instance.run(`CREATE TABLE attachment_downloads( - id STRING primary key, - timestamp INTEGER, - pending INTEGER, - json TEXT - );`); + db.transaction(() => { + db.exec(` + CREATE TABLE attachment_downloads( + id STRING primary key, + timestamp INTEGER, + pending INTEGER, + json TEXT + ); - await instance.run(`CREATE INDEX attachment_downloads_timestamp - ON attachment_downloads ( - timestamp - ) WHERE pending = 0;`); - await instance.run(`CREATE INDEX attachment_downloads_pending - ON attachment_downloads ( - pending - ) WHERE pending != 0;`); + CREATE INDEX attachment_downloads_timestamp + ON attachment_downloads ( + timestamp + ) WHERE pending = 0; + CREATE INDEX attachment_downloads_pending + ON attachment_downloads ( + pending + ) WHERE pending != 0; + `); - await instance.run('PRAGMA user_version = 9;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion9: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 9'); + })(); + + console.log('updateToSchemaVersion9: success!'); } -async function updateToSchemaVersion10( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion10(currentVersion: number, db: Database): void { if (currentVersion >= 10) { return; } console.log('updateToSchemaVersion10: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + DROP INDEX unprocessed_id; + DROP INDEX unprocessed_timestamp; + ALTER TABLE unprocessed RENAME TO unprocessed_old; - try { - await instance.run('DROP INDEX unprocessed_id;'); - await instance.run('DROP INDEX unprocessed_timestamp;'); - await instance.run('ALTER TABLE unprocessed RENAME TO unprocessed_old;'); + CREATE TABLE unprocessed( + id STRING, + timestamp INTEGER, + version INTEGER, + attempts INTEGER, + envelope TEXT, + decrypted TEXT, + source TEXT, + sourceDevice TEXT, + serverTimestamp INTEGER + ); - await instance.run(`CREATE TABLE unprocessed( - id STRING, - timestamp INTEGER, - version INTEGER, - attempts INTEGER, - envelope TEXT, - decrypted TEXT, - source TEXT, - sourceDevice TEXT, - serverTimestamp INTEGER - );`); + CREATE INDEX unprocessed_id ON unprocessed ( + id + ); + CREATE INDEX unprocessed_timestamp ON unprocessed ( + timestamp + ); - await instance.run(`CREATE INDEX unprocessed_id ON unprocessed ( - id - );`); - await instance.run(`CREATE INDEX unprocessed_timestamp ON unprocessed ( - timestamp - );`); + INSERT INTO unprocessed ( + id, + timestamp, + version, + attempts, + envelope, + decrypted, + source, + sourceDevice, + serverTimestamp + ) SELECT + id, + timestamp, + json_extract(json, '$.version'), + json_extract(json, '$.attempts'), + json_extract(json, '$.envelope'), + json_extract(json, '$.decrypted'), + json_extract(json, '$.source'), + json_extract(json, '$.sourceDevice'), + json_extract(json, '$.serverTimestamp') + FROM unprocessed_old; - await instance.run(`INSERT INTO unprocessed ( - id, - timestamp, - version, - attempts, - envelope, - decrypted, - source, - sourceDevice, - serverTimestamp - ) SELECT - id, - timestamp, - json_extract(json, '$.version'), - json_extract(json, '$.attempts'), - json_extract(json, '$.envelope'), - json_extract(json, '$.decrypted'), - json_extract(json, '$.source'), - json_extract(json, '$.sourceDevice'), - json_extract(json, '$.serverTimestamp') - FROM unprocessed_old; + DROP TABLE unprocessed_old; `); - await instance.run('DROP TABLE unprocessed_old;'); - - await instance.run('PRAGMA user_version = 10;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion10: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 10'); + })(); + console.log('updateToSchemaVersion10: success!'); } -async function updateToSchemaVersion11( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion11(currentVersion: number, db: Database): void { if (currentVersion >= 11) { return; } console.log('updateToSchemaVersion11: starting...'); - await instance.run('BEGIN TRANSACTION;'); - try { - await instance.run('DROP TABLE groups;'); + db.transaction(() => { + db.exec(` + DROP TABLE groups; + `); - await instance.run('PRAGMA user_version = 11;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion11: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 11'); + })(); + console.log('updateToSchemaVersion11: success!'); } -async function updateToSchemaVersion12( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion12(currentVersion: number, db: Database): void { if (currentVersion >= 12) { return; } console.log('updateToSchemaVersion12: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + CREATE TABLE sticker_packs( + id TEXT PRIMARY KEY, + key TEXT NOT NULL, - try { - await instance.run(`CREATE TABLE sticker_packs( - id TEXT PRIMARY KEY, - key TEXT NOT NULL, + author STRING, + coverStickerId INTEGER, + createdAt INTEGER, + downloadAttempts INTEGER, + installedAt INTEGER, + lastUsed INTEGER, + status STRING, + stickerCount INTEGER, + title STRING + ); - author STRING, - coverStickerId INTEGER, - createdAt INTEGER, - downloadAttempts INTEGER, - installedAt INTEGER, - lastUsed INTEGER, - status STRING, - stickerCount INTEGER, - title STRING - );`); + CREATE TABLE stickers( + id INTEGER NOT NULL, + packId TEXT NOT NULL, - await instance.run(`CREATE TABLE stickers( - id INTEGER NOT NULL, - packId TEXT NOT NULL, + emoji STRING, + height INTEGER, + isCoverOnly INTEGER, + lastUsed INTEGER, + path STRING, + width INTEGER, - emoji STRING, - height INTEGER, - isCoverOnly INTEGER, - lastUsed INTEGER, - path STRING, - width INTEGER, + PRIMARY KEY (id, packId), + CONSTRAINT stickers_fk + FOREIGN KEY (packId) + REFERENCES sticker_packs(id) + ON DELETE CASCADE + ); - PRIMARY KEY (id, packId), - CONSTRAINT stickers_fk - FOREIGN KEY (packId) - REFERENCES sticker_packs(id) - ON DELETE CASCADE - );`); + CREATE INDEX stickers_recents + ON stickers ( + lastUsed + ) WHERE lastUsed IS NOT NULL; - await instance.run(`CREATE INDEX stickers_recents - ON stickers ( - lastUsed - ) WHERE lastUsed IS NOT NULL;`); + CREATE TABLE sticker_references( + messageId STRING, + packId TEXT, + CONSTRAINT sticker_references_fk + FOREIGN KEY(packId) + REFERENCES sticker_packs(id) + ON DELETE CASCADE + ); + `); - await instance.run(`CREATE TABLE sticker_references( - messageId STRING, - packId TEXT, - CONSTRAINT sticker_references_fk - FOREIGN KEY(packId) - REFERENCES sticker_packs(id) - ON DELETE CASCADE - );`); - - await instance.run('PRAGMA user_version = 12;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion12: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 12'); + })(); + console.log('updateToSchemaVersion12: success!'); } -async function updateToSchemaVersion13( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion13(currentVersion: number, db: Database): void { if (currentVersion >= 13) { return; } console.log('updateToSchemaVersion13: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + ALTER TABLE sticker_packs ADD COLUMN attemptedStatus STRING; + `); - try { - await instance.run( - 'ALTER TABLE sticker_packs ADD COLUMN attemptedStatus STRING;' - ); - - await instance.run('PRAGMA user_version = 13;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion13: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 13'); + })(); + console.log('updateToSchemaVersion13: success!'); } -async function updateToSchemaVersion14( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion14(currentVersion: number, db: Database): void { if (currentVersion >= 14) { return; } console.log('updateToSchemaVersion14: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + CREATE TABLE emojis( + shortName STRING PRIMARY KEY, + lastUsage INTEGER + ); - try { - await instance.run(`CREATE TABLE emojis( - shortName STRING PRIMARY KEY, - lastUsage INTEGER - );`); + CREATE INDEX emojis_lastUsage + ON emojis ( + lastUsage + ); + `); - await instance.run(`CREATE INDEX emojis_lastUsage - ON emojis ( - lastUsage - );`); + db.pragma('user_version = 14'); + })(); - await instance.run('PRAGMA user_version = 14;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion14: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + console.log('updateToSchemaVersion14: success!'); } -async function updateToSchemaVersion15( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion15(currentVersion: number, db: Database): void { if (currentVersion >= 15) { return; } console.log('updateToSchemaVersion15: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + -- SQLite has again coerced our STRINGs into numbers, so we force it with TEXT + -- We create a new table then copy the data into it, since we can't modify columns - try { - // SQLite has again coerced our STRINGs into numbers, so we force it with TEXT - // We create a new table then copy the data into it, since we can't modify columns + DROP INDEX emojis_lastUsage; + ALTER TABLE emojis RENAME TO emojis_old; - await instance.run('DROP INDEX emojis_lastUsage;'); - await instance.run('ALTER TABLE emojis RENAME TO emojis_old;'); + CREATE TABLE emojis( + shortName TEXT PRIMARY KEY, + lastUsage INTEGER + ); + CREATE INDEX emojis_lastUsage + ON emojis ( + lastUsage + ); - await instance.run(`CREATE TABLE emojis( - shortName TEXT PRIMARY KEY, - lastUsage INTEGER - );`); - await instance.run(`CREATE INDEX emojis_lastUsage - ON emojis ( - lastUsage - );`); + DELETE FROM emojis WHERE shortName = 1; + INSERT INTO emojis(shortName, lastUsage) + SELECT shortName, lastUsage FROM emojis_old; - await instance.run('DELETE FROM emojis WHERE shortName = 1'); - await instance.run(`INSERT INTO emojis(shortName, lastUsage) - SELECT shortName, lastUsage FROM emojis_old; + DROP TABLE emojis_old; `); - await instance.run('DROP TABLE emojis_old;'); - - await instance.run('PRAGMA user_version = 15;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion15: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 15'); + })(); + console.log('updateToSchemaVersion15: success!'); } -async function updateToSchemaVersion16( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion16(currentVersion: number, db: Database): void { if (currentVersion >= 16) { return; } console.log('updateToSchemaVersion16: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + ALTER TABLE messages + ADD COLUMN messageTimer INTEGER; + ALTER TABLE messages + ADD COLUMN messageTimerStart INTEGER; + ALTER TABLE messages + ADD COLUMN messageTimerExpiresAt INTEGER; + ALTER TABLE messages + ADD COLUMN isErased INTEGER; - try { - await instance.run( - `ALTER TABLE messages - ADD COLUMN messageTimer INTEGER;` - ); - await instance.run( - `ALTER TABLE messages - ADD COLUMN messageTimerStart INTEGER;` - ); - await instance.run( - `ALTER TABLE messages - ADD COLUMN messageTimerExpiresAt INTEGER;` - ); - await instance.run( - `ALTER TABLE messages - ADD COLUMN isErased INTEGER;` - ); + CREATE INDEX messages_message_timer ON messages ( + messageTimer, + messageTimerStart, + messageTimerExpiresAt, + isErased + ) WHERE messageTimer IS NOT NULL; - await instance.run(`CREATE INDEX messages_message_timer ON messages ( - messageTimer, - messageTimerStart, - messageTimerExpiresAt, - isErased - ) WHERE messageTimer IS NOT NULL;`); + -- Updating full-text triggers to avoid anything with a messageTimer set - // Updating full-text triggers to avoid anything with a messageTimer set + DROP TRIGGER messages_on_insert; + DROP TRIGGER messages_on_delete; + DROP TRIGGER messages_on_update; - await instance.run('DROP TRIGGER messages_on_insert;'); - await instance.run('DROP TRIGGER messages_on_delete;'); - await instance.run('DROP TRIGGER messages_on_update;'); - - await instance.run(` CREATE TRIGGER messages_on_insert AFTER INSERT ON messages WHEN new.messageTimer IS NULL BEGIN @@ -1184,13 +965,9 @@ async function updateToSchemaVersion16( new.body ); END; - `); - await instance.run(` CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN DELETE FROM messages_fts WHERE id = old.id; END; - `); - await instance.run(` CREATE TRIGGER messages_on_update AFTER UPDATE ON messages WHEN new.messageTimer IS NULL BEGIN @@ -1205,34 +982,25 @@ async function updateToSchemaVersion16( END; `); - await instance.run('PRAGMA user_version = 16;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion16: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 16'); + })(); + console.log('updateToSchemaVersion16: success!'); } -async function updateToSchemaVersion17( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion17(currentVersion: number, db: Database): void { if (currentVersion >= 17) { return; } console.log('updateToSchemaVersion17: starting...'); - await instance.run('BEGIN TRANSACTION;'); - - try { + db.transaction(() => { try { - await instance.run( - `ALTER TABLE messages - ADD COLUMN isViewOnce INTEGER;` - ); + db.exec(` + ALTER TABLE messages + ADD COLUMN isViewOnce INTEGER; - await instance.run('DROP INDEX messages_message_timer;'); + DROP INDEX messages_message_timer; + `); } catch (error) { console.log( 'updateToSchemaVersion17: Message table already had isViewOnce column' @@ -1240,22 +1008,23 @@ async function updateToSchemaVersion17( } try { - await instance.run('DROP INDEX messages_view_once;'); + db.exec('DROP INDEX messages_view_once;'); } catch (error) { console.log( 'updateToSchemaVersion17: Index messages_view_once did not already exist' ); } - await instance.run(`CREATE INDEX messages_view_once ON messages ( - isErased - ) WHERE isViewOnce = 1;`); - // Updating full-text triggers to avoid anything with isViewOnce = 1 + db.exec(` + CREATE INDEX messages_view_once ON messages ( + isErased + ) WHERE isViewOnce = 1; - await instance.run('DROP TRIGGER messages_on_insert;'); - await instance.run('DROP TRIGGER messages_on_update;'); + -- Updating full-text triggers to avoid anything with isViewOnce = 1 + + DROP TRIGGER messages_on_insert; + DROP TRIGGER messages_on_update; - await instance.run(` CREATE TRIGGER messages_on_insert AFTER INSERT ON messages WHEN new.isViewOnce != 1 BEGIN @@ -1267,8 +1036,6 @@ async function updateToSchemaVersion17( new.body ); END; - `); - await instance.run(` CREATE TRIGGER messages_on_update AFTER UPDATE ON messages WHEN new.isViewOnce != 1 BEGIN @@ -1283,45 +1050,32 @@ async function updateToSchemaVersion17( END; `); - await instance.run('PRAGMA user_version = 17;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion17: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 17'); + })(); + console.log('updateToSchemaVersion17: success!'); } -async function updateToSchemaVersion18( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion18(currentVersion: number, db: Database): void { if (currentVersion >= 18) { return; } console.log('updateToSchemaVersion18: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + -- Delete and rebuild full-text search index to capture everything - try { - // Delete and rebuild full-text search index to capture everything + DELETE FROM messages_fts; + INSERT INTO messages_fts(messages_fts) VALUES('rebuild'); - await instance.run('DELETE FROM messages_fts;'); - await instance.run( - "INSERT INTO messages_fts(messages_fts) VALUES('rebuild');" - ); - - await instance.run(` INSERT INTO messages_fts(id, body) SELECT id, body FROM messages WHERE isViewOnce IS NULL OR isViewOnce != 1; - `); - // Fixing full-text triggers + -- Fixing full-text triggers - await instance.run('DROP TRIGGER messages_on_insert;'); - await instance.run('DROP TRIGGER messages_on_update;'); + DROP TRIGGER messages_on_insert; + DROP TRIGGER messages_on_update; - await instance.run(` CREATE TRIGGER messages_on_insert AFTER INSERT ON messages WHEN new.isViewOnce IS NULL OR new.isViewOnce != 1 BEGIN @@ -1333,8 +1087,6 @@ async function updateToSchemaVersion18( new.body ); END; - `); - await instance.run(` CREATE TRIGGER messages_on_update AFTER UPDATE ON messages WHEN new.isViewOnce IS NULL OR new.isViewOnce != 1 BEGIN @@ -1349,126 +1101,93 @@ async function updateToSchemaVersion18( END; `); - await instance.run('PRAGMA user_version = 18;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion18: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 18'); + })(); + console.log('updateToSchemaVersion18: success!'); } -async function updateToSchemaVersion19( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion19(currentVersion: number, db: Database): void { if (currentVersion >= 19) { return; } console.log('updateToSchemaVersion19: starting...'); - await instance.run('BEGIN TRANSACTION;'); + db.transaction(() => { + db.exec(` + ALTER TABLE conversations + ADD COLUMN profileFamilyName TEXT; + ALTER TABLE conversations + ADD COLUMN profileFullName TEXT; - await instance.run( - `ALTER TABLE conversations - ADD COLUMN profileFamilyName TEXT;` - ); - await instance.run( - `ALTER TABLE conversations - ADD COLUMN profileFullName TEXT;` - ); + -- Preload new field with the profileName we already have + UPDATE conversations SET profileFullName = profileName; + `); - // Preload new field with the profileName we already have - await instance.run('UPDATE conversations SET profileFullName = profileName'); + db.pragma('user_version = 19'); + })(); - try { - await instance.run('PRAGMA user_version = 19;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion19: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + console.log('updateToSchemaVersion19: success!'); } -async function updateToSchemaVersion20( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion20(currentVersion: number, db: Database): void { if (currentVersion >= 20) { return; } console.log('updateToSchemaVersion20: starting...'); - await instance.run('BEGIN TRANSACTION;'); - - try { - const migrationJobQueue = new PQueue({ - concurrency: 10, - timeout: 1000 * 60 * 5, - throwOnTimeout: true, - }); + db.transaction(() => { // The triggers on the messages table slow down this migration // significantly, so we drop them and recreate them later. // Drop triggers - const triggers = await instance.all( - 'SELECT * FROM sqlite_master WHERE type = "trigger" AND tbl_name = "messages"' - ); + const triggers = db + .prepare( + 'SELECT * FROM sqlite_master WHERE type = "trigger" AND tbl_name = "messages"' + ) + .all(); for (const trigger of triggers) { - await instance.run(`DROP TRIGGER ${trigger.name}`); + db.exec(`DROP TRIGGER ${trigger.name}`); } // Create new columns and indices - await instance.run('ALTER TABLE conversations ADD COLUMN e164 TEXT;'); - await instance.run('ALTER TABLE conversations ADD COLUMN uuid TEXT;'); - await instance.run('ALTER TABLE conversations ADD COLUMN groupId TEXT;'); - await instance.run('ALTER TABLE messages ADD COLUMN sourceUuid TEXT;'); - await instance.run( - 'ALTER TABLE sessions RENAME COLUMN number TO conversationId;' - ); - await instance.run( - 'CREATE INDEX conversations_e164 ON conversations(e164);' - ); - await instance.run( - 'CREATE INDEX conversations_uuid ON conversations(uuid);' - ); - await instance.run( - 'CREATE INDEX conversations_groupId ON conversations(groupId);' - ); - await instance.run( - 'CREATE INDEX messages_sourceUuid on messages(sourceUuid);' - ); + db.exec(` + ALTER TABLE conversations ADD COLUMN e164 TEXT; + ALTER TABLE conversations ADD COLUMN uuid TEXT; + ALTER TABLE conversations ADD COLUMN groupId TEXT; + ALTER TABLE messages ADD COLUMN sourceUuid TEXT; + ALTER TABLE sessions RENAME COLUMN number TO conversationId; + CREATE INDEX conversations_e164 ON conversations(e164); + CREATE INDEX conversations_uuid ON conversations(uuid); + CREATE INDEX conversations_groupId ON conversations(groupId); + CREATE INDEX messages_sourceUuid on messages(sourceUuid); - // Migrate existing IDs - await instance.run( - "UPDATE conversations SET e164 = '+' || id WHERE type = 'private';" - ); - await instance.run( - "UPDATE conversations SET groupId = id WHERE type = 'group';" - ); + -- Migrate existing IDs + UPDATE conversations SET e164 = '+' || id WHERE type = 'private'; + UPDATE conversations SET groupId = id WHERE type = 'group'; + `); // Drop invalid groups and any associated messages - const maybeInvalidGroups = await instance.all( - "SELECT * FROM conversations WHERE type = 'group' AND members IS NULL;" - ); + const maybeInvalidGroups = db + .prepare( + "SELECT * FROM conversations WHERE type = 'group' AND members IS NULL;" + ) + .all(); for (const group of maybeInvalidGroups) { - const json = JSON.parse(group.json); + const json: { id: string; members: Array } = JSON.parse(group.json); if (!json.members || !json.members.length) { - await instance.run('DELETE FROM conversations WHERE id = $id;', { - $id: json.id, + db.prepare('DELETE FROM conversations WHERE id = $id;').run({ + id: json.id, }); - await instance.run('DELETE FROM messages WHERE conversationId = $id;', { - $id: json.id, - }); - // await instance.run('DELETE FROM sessions WHERE conversationId = $id;', { - // $id: json.id, - // }); + db.prepare( + 'DELETE FROM messages WHERE conversationId = $id;' + ).run({ id: json.id }); } } // Generate new IDs and alter data - const allConversations = await instance.all('SELECT * FROM conversations;'); + const allConversations = db + .prepare('SELECT * FROM conversations;') + .all(); const allConversationsByOldId = keyBy(allConversations, 'id'); for (const row of allConversations) { @@ -1483,71 +1202,91 @@ async function updateToSchemaVersion20( } const patch = JSON.stringify(patchObj); - await instance.run( - 'UPDATE conversations SET id = $newId, json = JSON_PATCH(json, $patch) WHERE id = $oldId', - { - $newId: newId, - $oldId: oldId, - $patch: patch, - } - ); + db.prepare( + ` + UPDATE conversations + SET id = $newId, json = JSON_PATCH(json, $patch) + WHERE id = $oldId + ` + ).run({ + newId, + oldId, + patch, + }); const messagePatch = JSON.stringify({ conversationId: newId }); - await instance.run( - 'UPDATE messages SET conversationId = $newId, json = JSON_PATCH(json, $patch) WHERE conversationId = $oldId', - { $newId: newId, $oldId: oldId, $patch: messagePatch } - ); + db.prepare( + ` + UPDATE messages + SET conversationId = $newId, json = JSON_PATCH(json, $patch) + WHERE conversationId = $oldId + ` + ).run({ newId, oldId, patch: messagePatch }); } - const groupConverations = await instance.all( - "SELECT * FROM conversations WHERE type = 'group';" - ); + const groupConversations: Array<{ + id: string; + members: string; + json: string; + }> = db + .prepare( + ` + SELECT id, members, json FROM conversations WHERE type = 'group'; + ` + ) + .all(); // Update group conversations, point members at new conversation ids - migrationJobQueue.addAll( - groupConverations.map(groupRow => async () => { - const members = groupRow.members.split(/\s?\+/).filter(Boolean); - const newMembers = []; - for (const m of members) { - const memberRow = allConversationsByOldId[m]; + groupConversations.forEach(groupRow => { + const members = groupRow.members.split(/\s?\+/).filter(Boolean); + const newMembers = []; + for (const m of members) { + const memberRow = allConversationsByOldId[m]; - if (memberRow) { - newMembers.push(memberRow.id); - } else { - // We didn't previously have a private conversation for this member, - // we need to create one - const id = generateUUID(); - await saveConversation( - { - id, - e164: m, - type: 'private', - version: 2, - unreadCount: 0, - verified: 0, - }, - instance - ); + if (memberRow) { + newMembers.push(memberRow.id); + } else { + // We didn't previously have a private conversation for this member, + // we need to create one + const id = generateUUID(); + saveConversation({ + id, + e164: m, + type: 'private', + version: 2, + unreadCount: 0, + verified: 0, - newMembers.push(id); - } + // Not directly used by saveConversation, but are necessary + // for conversation model + inbox_position: 0, + isPinned: false, + lastMessageDeletedForEveryone: false, + markedUnread: false, + messageCount: 0, + sentMessageCount: 0, + profileSharing: false, + }); + + newMembers.push(id); } - const json = { ...jsonToObject(groupRow.json), members: newMembers }; - const newMembersValue = newMembers.join(' '); - await instance.run( - 'UPDATE conversations SET members = $newMembersValue, json = $newJsonValue WHERE id = $id', - { - $id: groupRow.id, - $newMembersValue: newMembersValue, - $newJsonValue: objectToJSON(json), - } - ); - }) - ); - // Wait for group conversation updates to finish - await migrationJobQueue.onEmpty(); + } + const json = { ...jsonToObject(groupRow.json), members: newMembers }; + const newMembersValue = newMembers.join(' '); + db.prepare( + ` + UPDATE conversations + SET members = $newMembersValue, json = $newJsonValue + WHERE id = $id + ` + ).run({ + id: groupRow.id, + newMembersValue, + newJsonValue: objectToJSON(json), + }); + }); // Update sessions to stable IDs - const allSessions = await instance.all('SELECT * FROM sessions;'); + const allSessions = db.prepare('SELECT * FROM sessions;').all(); for (const session of allSessions) { // Not using patch here so we can explicitly delete a property rather than // implicitly delete via null @@ -1558,72 +1297,63 @@ async function updateToSchemaVersion20( newJson.id = `${newJson.conversationId}.${newJson.deviceId}`; } delete newJson.number; - await instance.run( + db.prepare( ` UPDATE sessions SET id = $newId, json = $newJson, conversationId = $newConversationId WHERE id = $oldId - `, - { - $newId: newJson.id, - $newJson: objectToJSON(newJson), - $oldId: session.id, - $newConversationId: newJson.conversationId, - } - ); + ` + ).run({ + newId: newJson.id, + newJson: objectToJSON(newJson), + oldId: session.id, + newConversationId: newJson.conversationId, + }); } // Update identity keys to stable IDs - const allIdentityKeys = await instance.all('SELECT * FROM identityKeys;'); + const allIdentityKeys = db + .prepare('SELECT * FROM identityKeys;') + .all(); for (const identityKey of allIdentityKeys) { const newJson = JSON.parse(identityKey.json); newJson.id = allConversationsByOldId[newJson.id]; - await instance.run( + db.prepare( ` UPDATE identityKeys SET id = $newId, json = $newJson WHERE id = $oldId - `, - { - $newId: newJson.id, - $newJson: objectToJSON(newJson), - $oldId: identityKey.id, - } - ); + ` + ).run({ + newId: newJson.id, + newJson: objectToJSON(newJson), + oldId: identityKey.id, + }); } // Recreate triggers for (const trigger of triggers) { - await instance.run(trigger.sql); + db.exec(trigger.sql); } - await instance.run('PRAGMA user_version = 20;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion20: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 20'); + })(); + console.log('updateToSchemaVersion20: success!'); } -async function updateToSchemaVersion21( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion21(currentVersion: number, db: Database): void { if (currentVersion >= 21) { return; } - try { - await instance.run('BEGIN TRANSACTION;'); - await instance.run(` + + db.transaction(() => { + db.exec(` UPDATE conversations SET json = json_set( json, '$.messageCount', (SELECT count(*) FROM messages WHERE messages.conversationId = conversations.id) ); - `); - await instance.run(` UPDATE conversations SET json = json_set( json, @@ -1631,86 +1361,59 @@ async function updateToSchemaVersion21( (SELECT count(*) FROM messages WHERE messages.conversationId = conversations.id AND messages.type = 'outgoing') ); `); - await instance.run('PRAGMA user_version = 21;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion21: success!'); - } catch (error) { - await instance.run('ROLLBACK'); - throw error; - } + db.pragma('user_version = 21'); + })(); + console.log('updateToSchemaVersion21: success!'); } -async function updateToSchemaVersion22( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion22(currentVersion: number, db: Database): void { if (currentVersion >= 22) { return; } - try { - await instance.run('BEGIN TRANSACTION;'); - await instance.run( - `ALTER TABLE unprocessed - ADD COLUMN sourceUuid STRING;` - ); - await instance.run('PRAGMA user_version = 22;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion22: success!'); - } catch (error) { - await instance.run('ROLLBACK'); - throw error; - } + db.transaction(() => { + db.exec(` + ALTER TABLE unprocessed + ADD COLUMN sourceUuid STRING; + `); + + db.pragma('user_version = 22'); + })(); + console.log('updateToSchemaVersion22: success!'); } -async function updateToSchemaVersion23( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion23(currentVersion: number, db: Database): void { if (currentVersion >= 23) { return; } - try { - await instance.run('BEGIN TRANSACTION;'); - // Remove triggers which keep full-text search up to date - await instance.run('DROP TRIGGER messages_on_insert;'); - await instance.run('DROP TRIGGER messages_on_update;'); - await instance.run('DROP TRIGGER messages_on_delete;'); + db.transaction(() => { + db.exec(` + -- Remove triggers which keep full-text search up to date + DROP TRIGGER messages_on_insert; + DROP TRIGGER messages_on_update; + DROP TRIGGER messages_on_delete; + `); - await instance.run('PRAGMA user_version = 23;'); - await instance.run('COMMIT TRANSACTION;'); - console.log('updateToSchemaVersion23: success!'); - } catch (error) { - await instance.run('ROLLBACK'); - throw error; - } + db.pragma('user_version = 23'); + })(); + console.log('updateToSchemaVersion23: success!'); } -async function updateToSchemaVersion24( - currentVersion: number, - instance: PromisifiedSQLDatabase -) { +function updateToSchemaVersion24(currentVersion: number, db: Database): void { if (currentVersion >= 24) { return; } - await instance.run('BEGIN TRANSACTION;'); - - try { - await instance.run(` + db.transaction(() => { + db.exec(` ALTER TABLE conversations ADD COLUMN profileLastFetchedAt INTEGER; `); - await instance.run('PRAGMA user_version = 24;'); - await instance.run('COMMIT TRANSACTION;'); - - console.log('updateToSchemaVersion24: success!'); - } catch (error) { - await instance.run('ROLLBACK;'); - throw error; - } + db.pragma('user_version = 24'); + })(); + console.log('updateToSchemaVersion24: success!'); } const SCHEMA_VERSIONS = [ @@ -1718,7 +1421,7 @@ const SCHEMA_VERSIONS = [ updateToSchemaVersion2, updateToSchemaVersion3, updateToSchemaVersion4, - (_v: number, _i: PromisifiedSQLDatabase) => null, // version 5 was dropped + (_v: number, _i: Database) => null, // version 5 was dropped updateToSchemaVersion6, updateToSchemaVersion7, updateToSchemaVersion8, @@ -1740,12 +1443,12 @@ const SCHEMA_VERSIONS = [ updateToSchemaVersion24, ]; -async function updateSchema(instance: PromisifiedSQLDatabase) { - const sqliteVersion = await getSQLiteVersion(instance); - const sqlcipherVersion = await getSQLCipherVersion(instance); - const userVersion = await getUserVersion(instance); +function updateSchema(db: Database): void { + const sqliteVersion = getSQLiteVersion(db); + const sqlcipherVersion = getSQLCipherVersion(db); + const userVersion = getUserVersion(db); const maxUserVersion = SCHEMA_VERSIONS.length; - const schemaVersion = await getSchemaVersion(instance); + const schemaVersion = getSchemaVersion(db); console.log( 'updateSchema:\n', @@ -1766,23 +1469,21 @@ async function updateSchema(instance: PromisifiedSQLDatabase) { const runSchemaUpdate = SCHEMA_VERSIONS[index]; // Yes, we really want to do this asynchronously, in order - await runSchemaUpdate(userVersion, instance); + runSchemaUpdate(userVersion, db); } } -let globalInstance: PromisifiedSQLDatabase | undefined; -let globalInstanceRenderer: PromisifiedSQLDatabase | undefined; +let globalInstance: Database | undefined; +let globalInstanceRenderer: Database | undefined; let databaseFilePath: string | undefined; let indexedDBPath: string | undefined; async function initialize({ configDir, key, - messages, }: { configDir: string; key: string; - messages: LocaleMessagesType; }) { if (globalInstance) { throw new Error('Cannot initialize more than once!'); @@ -1794,9 +1495,6 @@ async function initialize({ if (!isString(key)) { throw new Error('initialize: key is required!'); } - if (!isObject(messages)) { - throw new Error('initialize: message is required!'); - } indexedDBPath = join(configDir, 'IndexedDB'); @@ -1805,37 +1503,19 @@ async function initialize({ databaseFilePath = join(dbDir, 'db.sqlite'); - let promisified: PromisifiedSQLDatabase | undefined; + let db: Database | undefined; try { - promisified = await openAndSetUpSQLCipher(databaseFilePath, { key }); + db = openAndSetUpSQLCipher(databaseFilePath, { key }); - // if (promisified) { - // promisified.on('trace', async statement => { - // if ( - // !globalInstance || - // statement.startsWith('--') || - // statement.includes('COMMIT') || - // statement.includes('BEGIN') || - // statement.includes('ROLLBACK') - // ) { - // return; - // } + // For profiling use: + // db.pragma('cipher_profile=\'sqlcipher.log\''); - // // Note that this causes problems when attempting to commit transactions - this - // // statement is running, and we get at SQLITE_BUSY error. So we delay. - // await new Promise(resolve => setTimeout(resolve, 1000)); - - // const data = await promisified.get(`EXPLAIN QUERY PLAN ${statement}`); - // console._log(`EXPLAIN QUERY PLAN ${statement}\n`, data && data.detail); - // }); - // } - - await updateSchema(promisified); + updateSchema(db); // test database - const cipherIntegrityResult = await getSQLCipherIntegrityCheck(promisified); + const cipherIntegrityResult = getSQLCipherIntegrityCheck(db); if (cipherIntegrityResult) { console.log( 'Database cipher integrity check failed:', @@ -1845,21 +1525,21 @@ async function initialize({ `Cipher integrity check failed: ${cipherIntegrityResult}` ); } - const integrityResult = await getSQLIntegrityCheck(promisified); + const integrityResult = getSQLIntegrityCheck(db); if (integrityResult) { console.log('Database integrity check failed:', integrityResult); throw new Error(`Integrity check failed: ${integrityResult}`); } // At this point we can allow general access to the database - globalInstance = promisified; + globalInstance = db; // test database - await getMessageCount(); + getMessageCount(); } catch (error) { console.log('Database startup error:', error.stack); - if (promisified) { - await promisified.close(); + if (db) { + db.close(); } throw error; } @@ -1895,33 +1575,33 @@ async function initializeRenderer({ databaseFilePath = join(dbDir, 'db.sqlite'); } - let promisified: PromisifiedSQLDatabase | undefined; + let promisified: Database | undefined; try { - promisified = await openAndSetUpSQLCipher(databaseFilePath, { key }); + promisified = openAndSetUpSQLCipher(databaseFilePath, { key }); // At this point we can allow general access to the database globalInstanceRenderer = promisified; // test database - await getMessageCount(); + getMessageCount(); } catch (error) { window.log.error('Database startup error:', error.stack); throw error; } } -async function close() { +async function close(): Promise { if (!globalInstance) { return; } const dbRef = globalInstance; globalInstance = undefined; - await dbRef.close(); + dbRef.close(); } -async function removeDB() { +async function removeDB(): Promise { if (globalInstance) { throw new Error('removeDB: Cannot erase database when it is open!'); } @@ -1936,7 +1616,7 @@ async function removeDB() { rimraf.sync(`${databaseFilePath}-wal`); } -async function removeIndexedDBFiles() { +async function removeIndexedDBFiles(): Promise { if (!indexedDBPath) { throw new Error( 'removeIndexedDBFiles: Need to initialize and set indexedDBPath first!' @@ -1948,7 +1628,7 @@ async function removeIndexedDBFiles() { indexedDBPath = undefined; } -function getInstance(): PromisifiedSQLDatabase { +function getInstance(): Database { if (isRenderer()) { if (!globalInstanceRenderer) { throw new Error('getInstance: globalInstanceRenderer not set!'); @@ -1964,97 +1644,105 @@ function getInstance(): PromisifiedSQLDatabase { } const IDENTITY_KEYS_TABLE = 'identityKeys'; -async function createOrUpdateIdentityKey(data: IdentityKeyType) { +function createOrUpdateIdentityKey(data: IdentityKeyType): Promise { return createOrUpdate(IDENTITY_KEYS_TABLE, data); } -async function getIdentityKeyById(id: string) { +function getIdentityKeyById(id: string): Promise { return getById(IDENTITY_KEYS_TABLE, id); } -async function bulkAddIdentityKeys(array: Array) { +function bulkAddIdentityKeys(array: Array): Promise { return bulkAdd(IDENTITY_KEYS_TABLE, array); } -bulkAddIdentityKeys.needsSerial = true; -async function removeIdentityKeyById(id: string) { +function removeIdentityKeyById(id: string): Promise { return removeById(IDENTITY_KEYS_TABLE, id); } -async function removeAllIdentityKeys() { +function removeAllIdentityKeys(): Promise { return removeAllFromTable(IDENTITY_KEYS_TABLE); } -async function getAllIdentityKeys() { +function getAllIdentityKeys(): Promise> { return getAllFromTable(IDENTITY_KEYS_TABLE); } const PRE_KEYS_TABLE = 'preKeys'; -async function createOrUpdatePreKey(data: PreKeyType) { +function createOrUpdatePreKey(data: PreKeyType): Promise { return createOrUpdate(PRE_KEYS_TABLE, data); } -async function getPreKeyById(id: number) { +function getPreKeyById(id: number): Promise { return getById(PRE_KEYS_TABLE, id); } -async function bulkAddPreKeys(array: Array) { +function bulkAddPreKeys(array: Array): Promise { return bulkAdd(PRE_KEYS_TABLE, array); } -bulkAddPreKeys.needsSerial = true; -async function removePreKeyById(id: number) { +function removePreKeyById(id: number): Promise { return removeById(PRE_KEYS_TABLE, id); } -async function removeAllPreKeys() { +function removeAllPreKeys(): Promise { return removeAllFromTable(PRE_KEYS_TABLE); } -async function getAllPreKeys() { +function getAllPreKeys(): Promise> { return getAllFromTable(PRE_KEYS_TABLE); } const SIGNED_PRE_KEYS_TABLE = 'signedPreKeys'; -async function createOrUpdateSignedPreKey(data: SignedPreKeyType) { +function createOrUpdateSignedPreKey(data: SignedPreKeyType): Promise { return createOrUpdate(SIGNED_PRE_KEYS_TABLE, data); } -async function getSignedPreKeyById(id: number) { +function getSignedPreKeyById( + id: number +): Promise { return getById(SIGNED_PRE_KEYS_TABLE, id); } -async function bulkAddSignedPreKeys(array: Array) { +function bulkAddSignedPreKeys(array: Array): Promise { return bulkAdd(SIGNED_PRE_KEYS_TABLE, array); } -bulkAddSignedPreKeys.needsSerial = true; -async function removeSignedPreKeyById(id: number) { +function removeSignedPreKeyById(id: number): Promise { return removeById(SIGNED_PRE_KEYS_TABLE, id); } -async function removeAllSignedPreKeys() { +function removeAllSignedPreKeys(): Promise { return removeAllFromTable(SIGNED_PRE_KEYS_TABLE); } -async function getAllSignedPreKeys() { +async function getAllSignedPreKeys(): Promise> { const db = getInstance(); - const rows = await db.all('SELECT json FROM signedPreKeys ORDER BY id ASC;'); + const rows: JSONRows = db + .prepare( + ` + SELECT json + FROM signedPreKeys + ORDER BY id ASC; + ` + ) + .all(); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } const ITEMS_TABLE = 'items'; -async function createOrUpdateItem(data: ItemType) { +function createOrUpdateItem(data: ItemType): Promise { return createOrUpdate(ITEMS_TABLE, data); } -async function getItemById(id: string) { +function getItemById(id: string): Promise { return getById(ITEMS_TABLE, id); } -async function getAllItems() { +async function getAllItems(): Promise> { const db = getInstance(); - const rows = await db.all('SELECT json FROM items ORDER BY id ASC;'); + const rows: JSONRows = db + .prepare('SELECT json FROM items ORDER BY id ASC;') + .all(); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function bulkAddItems(array: Array) { +function bulkAddItems(array: Array): Promise { return bulkAdd(ITEMS_TABLE, array); } -bulkAddItems.needsSerial = true; -async function removeItemById(id: string) { +function removeItemById(id: string): Promise { return removeById(ITEMS_TABLE, id); } -async function removeAllItems() { +function removeAllItems(): Promise { return removeAllFromTable(ITEMS_TABLE); } const SESSIONS_TABLE = 'sessions'; -async function createOrUpdateSession(data: SessionType) { +async function createOrUpdateSession(data: SessionType): Promise { const db = getInstance(); const { id, conversationId } = data; if (!id) { @@ -2068,8 +1756,9 @@ async function createOrUpdateSession(data: SessionType) { ); } - await db.run( - `INSERT OR REPLACE INTO sessions ( + db.prepare( + ` + INSERT OR REPLACE INTO sessions ( id, conversationId, json @@ -2077,108 +1766,128 @@ async function createOrUpdateSession(data: SessionType) { $id, $conversationId, $json - )`, - { - $id: id, - $conversationId: conversationId, - $json: objectToJSON(data), - } - ); -} -async function createOrUpdateSessions(array: Array) { - const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - - try { - await Promise.all([ - ...map(array, async item => createOrUpdateSession(item)), - ]); - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } -} -createOrUpdateSessions.needsSerial = true; - -async function getSessionById(id: string) { - return getById(SESSIONS_TABLE, id); -} -async function getSessionsById(conversationId: string) { - const db = getInstance(); - const rows = await db.all( - 'SELECT * FROM sessions WHERE conversationId = $conversationId;', - { - $conversationId: conversationId, - } - ); - - return map(rows, row => jsonToObject(row.json)); -} -async function bulkAddSessions(array: Array) { - return bulkAdd(SESSIONS_TABLE, array); -} -bulkAddSessions.needsSerial = true; -async function removeSessionById(id: string) { - return removeById(SESSIONS_TABLE, id); -} -async function removeSessionsByConversation(conversationId: string) { - const db = getInstance(); - await db.run('DELETE FROM sessions WHERE conversationId = $conversationId;', { - $conversationId: conversationId, + ) + ` + ).run({ + id, + conversationId, + json: objectToJSON(data), }); } -async function removeAllSessions() { +async function createOrUpdateSessions( + array: Array +): Promise { + const db = getInstance(); + + db.transaction(() => { + for (const item of array) { + createOrUpdateSession(item); + } + })(); +} + +async function getSessionById(id: string): Promise { + return getById(SESSIONS_TABLE, id); +} +async function getSessionsById( + conversationId: string +): Promise> { + const db = getInstance(); + const rows: JSONRows = db + .prepare( + ` + SELECT json + FROM sessions + WHERE conversationId = $conversationId; + ` + ) + .all({ + conversationId, + }); + + return rows.map(row => jsonToObject(row.json)); +} +function bulkAddSessions(array: Array): Promise { + return bulkAdd(SESSIONS_TABLE, array); +} +function removeSessionById(id: string): Promise { + return removeById(SESSIONS_TABLE, id); +} +async function removeSessionsByConversation( + conversationId: string +): Promise { + const db = getInstance(); + db.prepare( + ` + DELETE FROM sessions + WHERE conversationId = $conversationId; + ` + ).run({ + conversationId, + }); +} +function removeAllSessions(): Promise { return removeAllFromTable(SESSIONS_TABLE); } -async function getAllSessions() { +function getAllSessions(): Promise> { return getAllFromTable(SESSIONS_TABLE); } -async function createOrUpdate(table: string, data: any) { +async function createOrUpdate( + table: string, + data: Record & { id: string | number } +): Promise { const db = getInstance(); const { id } = data; if (!id) { throw new Error('createOrUpdate: Provided data did not have a truthy id'); } - await db.run( - `INSERT OR REPLACE INTO ${table} ( + db.prepare( + ` + INSERT OR REPLACE INTO ${table} ( id, json ) values ( $id, $json - )`, - { - $id: id, - $json: objectToJSON(data), - } - ); -} - -async function bulkAdd(table: string, array: Array) { - const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - - try { - await Promise.all([ - ...map(array, async data => createOrUpdate(table, data)), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } -} -bulkAdd.needsSerial = true; - -async function getById(table: string, id: string | number) { - const db = getInstance(); - const row = await db.get(`SELECT * FROM ${table} WHERE id = $id;`, { - $id: id, + ) + ` + ).run({ + id, + json: objectToJSON(data), }); +} + +async function bulkAdd( + table: string, + array: Array & { id: string | number }> +): Promise { + const db = getInstance(); + + db.transaction(() => { + for (const data of array) { + createOrUpdate(table, data); + } + })(); +} + +async function getById( + table: string, + id: string | number +): Promise { + const db = getInstance(); + const row = db + .prepare( + ` + SELECT * + FROM ${table} + WHERE id = $id; + ` + ) + .get({ + id, + }); if (!row) { return undefined; @@ -2187,11 +1896,18 @@ async function getById(table: string, id: string | number) { return jsonToObject(row.json); } -async function removeById(table: string, id: string | number) { +async function removeById( + table: string, + id: string | number | Array +): Promise { const db = getInstance(); if (!Array.isArray(id)) { - await db.run(`DELETE FROM ${table} WHERE id = $id;`, { $id: id }); - + db.prepare( + ` + DELETE FROM ${table} + WHERE id = $id; + ` + ).run({ id }); return; } @@ -2200,29 +1916,35 @@ async function removeById(table: string, id: string | number) { } // Our node interface doesn't seem to allow you to replace one single ? with an array - await db.run( - `DELETE FROM ${table} WHERE id IN ( ${id.map(() => '?').join(', ')} );`, - id - ); + db.prepare( + ` + DELETE FROM ${table} + WHERE id IN ( ${id.map(() => '?').join(', ')} ); + ` + ).run(id); } -async function removeAllFromTable(table: string) { +async function removeAllFromTable(table: string): Promise { const db = getInstance(); - await db.run(`DELETE FROM ${table};`); + db.prepare(`DELETE FROM ${table};`).run(); } -async function getAllFromTable(table: string) { +async function getAllFromTable(table: string): Promise> { const db = getInstance(); - const rows = await db.all(`SELECT json FROM ${table};`); + const rows: JSONRows = db + .prepare(`SELECT json FROM ${table};`) + .all(); return rows.map(row => jsonToObject(row.json)); } // Conversations -async function getConversationCount() { +async function getConversationCount(): Promise { const db = getInstance(); - const row = await db.get('SELECT count(*) from conversations;'); + const row = db + .prepare('SELECT count(*) from conversations;') + .get(); if (!row) { throw new Error( @@ -2235,8 +1957,8 @@ async function getConversationCount() { async function saveConversation( data: ConversationType, - instance = getInstance() -) { + db = getInstance() +): Promise { const { active_at, e164, @@ -2259,82 +1981,74 @@ async function saveConversation( ? members.join(' ') : null; - await instance.run( - `INSERT INTO conversations ( + db.prepare( + ` + INSERT INTO conversations ( + id, + json, + + e164, + uuid, + groupId, + + active_at, + type, + members, + name, + profileName, + profileFamilyName, + profileFullName, + profileLastFetchedAt + ) values ( + $id, + $json, + + $e164, + $uuid, + $groupId, + + $active_at, + $type, + $members, + $name, + $profileName, + $profileFamilyName, + $profileFullName, + $profileLastFetchedAt + ); + ` + ).run({ id, - json, + json: objectToJSON(omit(data, ['profileLastFetchedAt'])), - e164, - uuid, - groupId, + e164: e164 || null, + uuid: uuid || null, + groupId: groupId || null, - active_at, + active_at: active_at || null, type, - members, - name, - profileName, - profileFamilyName, - profileFullName, - profileLastFetchedAt - ) values ( - $id, - $json, - - $e164, - $uuid, - $groupId, - - $active_at, - $type, - $members, - $name, - $profileName, - $profileFamilyName, - $profileFullName, - $profileLastFetchedAt - );`, - { - $id: id, - $json: objectToJSON(omit(data, ['profileLastFetchedAt'])), - - $e164: e164, - $uuid: uuid, - $groupId: groupId, - - $active_at: active_at, - $type: type, - $members: membersList, - $name: name, - $profileName: profileName, - $profileFamilyName: profileFamilyName, - $profileFullName: combineNames(profileName, profileFamilyName), - $profileLastFetchedAt: profileLastFetchedAt, - } - ); + members: membersList, + name: name || null, + profileName: profileName || null, + profileFamilyName: profileFamilyName || null, + profileFullName: combineNames(profileName, profileFamilyName) || null, + profileLastFetchedAt: profileLastFetchedAt || null, + }); } async function saveConversations( arrayOfConversations: Array -) { +): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - ...map(arrayOfConversations, async conversation => - saveConversation(conversation) - ), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + for (const conversation of arrayOfConversations) { + saveConversation(conversation); + } + })(); } -saveConversations.needsSerial = true; -async function updateConversation(data: ConversationType) { +async function updateConversation(data: ConversationType): Promise { const db = getInstance(); const { id, @@ -2357,8 +2071,9 @@ async function updateConversation(data: ConversationType) { ? members.join(' ') : null; - await db.run( - `UPDATE conversations SET + db.prepare( + ` + UPDATE conversations SET json = $json, e164 = $e164, @@ -2372,44 +2087,44 @@ async function updateConversation(data: ConversationType) { profileFamilyName = $profileFamilyName, profileFullName = $profileFullName, profileLastFetchedAt = $profileLastFetchedAt - WHERE id = $id;`, - { - $id: id, - $json: objectToJSON(omit(data, ['profileLastFetchedAt'])), + WHERE id = $id; + ` + ).run({ + id, + json: objectToJSON(omit(data, ['profileLastFetchedAt'])), - $e164: e164, - $uuid: uuid, + e164: e164 || null, + uuid: uuid || null, - $active_at: active_at, - $type: type, - $members: membersList, - $name: name, - $profileName: profileName, - $profileFamilyName: profileFamilyName, - $profileFullName: combineNames(profileName, profileFamilyName), - $profileLastFetchedAt: profileLastFetchedAt, - } - ); + active_at: active_at || null, + type, + members: membersList, + name: name || null, + profileName: profileName || null, + profileFamilyName: profileFamilyName || null, + profileFullName: combineNames(profileName, profileFamilyName) || null, + profileLastFetchedAt: profileLastFetchedAt || null, + }); } -async function updateConversations(array: Array) { +async function updateConversations( + array: Array +): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([...map(array, async item => updateConversation(item))]); - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + for (const item of array) { + updateConversation(item); + } + })(); } -updateConversations.needsSerial = true; -async function removeConversation(id: Array | string) { +async function removeConversation(id: Array | string): Promise { const db = getInstance(); if (!Array.isArray(id)) { - await db.run('DELETE FROM conversations WHERE id = $id;', { $id: id }); + db.prepare('DELETE FROM conversations WHERE id = id;').run({ + id, + }); return; } @@ -2419,81 +2134,104 @@ async function removeConversation(id: Array | string) { } // Our node interface doesn't seem to allow you to replace one single ? with an array - await db.run( - `DELETE FROM conversations WHERE id IN ( ${id - .map(() => '?') - .join(', ')} );`, - id - ); + db.prepare( + ` + DELETE FROM conversations + WHERE id IN ( ${id.map(() => '?').join(', ')} ); + ` + ).run(id); } -async function getConversationById(id: string) { +async function getConversationById( + id: string +): Promise { const db = getInstance(); - const row = await db.get('SELECT * FROM conversations WHERE id = $id;', { - $id: id, - }); + const row: { json: string } = db + .prepare('SELECT json FROM conversations WHERE id = $id;') + .get({ id }); if (!row) { - return null; + return undefined; } return jsonToObject(row.json); } -async function eraseStorageServiceStateFromConversations() { +async function eraseStorageServiceStateFromConversations(): Promise { const db = getInstance(); - await db.run( - `UPDATE conversations SET + db.prepare( + ` + UPDATE conversations + SET json = json_remove(json, '$.storageID', '$.needsStorageServiceSync', '$.unknownFields', '$.storageProfileKey'); ` - ); + ).run(); } -async function getAllConversations() { +async function getAllConversations(): Promise> { const db = getInstance(); - const rows = await db.all(` - SELECT json, profileLastFetchedAt - FROM conversations - ORDER BY id ASC; - `); + const rows: ConversationRows = db + .prepare( + ` + SELECT json, profileLastFetchedAt + FROM conversations + ORDER BY id ASC; + ` + ) + .all(); - return map(rows, row => rowToConversation(row)); + return rows.map(row => rowToConversation(row)); } -async function getAllConversationIds() { +async function getAllConversationIds(): Promise> { const db = getInstance(); - const rows = await db.all('SELECT id FROM conversations ORDER BY id ASC;'); + const rows: Array<{ id: string }> = db + .prepare( + ` + SELECT id FROM conversations ORDER BY id ASC; + ` + ) + .all(); - return map(rows, row => row.id); + return rows.map(row => row.id); } -async function getAllPrivateConversations() { +async function getAllPrivateConversations(): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json, profileLastFetchedAt - FROM conversations - WHERE type = 'private' - ORDER BY id ASC;` - ); + const rows: ConversationRows = db + .prepare( + ` + SELECT json, profileLastFetchedAt + FROM conversations + WHERE type = 'private' + ORDER BY id ASC; + ` + ) + .all(); - return map(rows, row => rowToConversation(row)); + return rows.map(row => rowToConversation(row)); } -async function getAllGroupsInvolvingId(id: string) { +async function getAllGroupsInvolvingId( + id: string +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json, profileLastFetchedAt - FROM conversations WHERE - type = 'group' AND - members LIKE $id - ORDER BY id ASC;`, - { - $id: `%${id}%`, - } - ); + const rows: ConversationRows = db + .prepare( + ` + SELECT json, profileLastFetchedAt + FROM conversations WHERE + type = 'group' AND + members LIKE $id + ORDER BY id ASC; + ` + ) + .all({ + id: `%${id}%`, + }); - return map(rows, row => rowToConversation(row)); + return rows.map(row => rowToConversation(row)); } async function searchConversations( @@ -2501,23 +2239,26 @@ async function searchConversations( { limit }: { limit?: number } = {} ): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json, profileLastFetchedAt - FROM conversations WHERE - ( - e164 LIKE $query OR - name LIKE $query OR - profileFullName LIKE $query - ) - ORDER BY active_at DESC - LIMIT $limit`, - { - $query: `%${query}%`, - $limit: limit || 100, - } - ); + const rows: ConversationRows = db + .prepare( + ` + SELECT json, profileLastFetchedAt + FROM conversations WHERE + ( + e164 LIKE $query OR + name LIKE $query OR + profileFullName LIKE $query + ) + ORDER BY active_at DESC + LIMIT $limit + ` + ) + .all({ + query: `%${query}%`, + limit: limit || 100, + }); - return map(rows, row => rowToConversation(row)); + return rows.map(row => rowToConversation(row)); } async function searchMessages( @@ -2525,26 +2266,26 @@ async function searchMessages( { limit }: { limit?: number } = {} ): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT - messages.json, - snippet(messages_fts, -1, '<>', '<>', '...', 10) as snippet - FROM messages_fts - INNER JOIN messages on messages_fts.id = messages.id - WHERE - messages_fts match $query - ORDER BY messages.received_at DESC, messages.sent_at DESC - LIMIT $limit;`, - { - $query: query, - $limit: limit || 500, - } - ); + const rows: Array = db + .prepare( + ` + SELECT + messages.json, + snippet(messages_fts, -1, '<>', '<>', '...', 10) as snippet + FROM messages_fts + INNER JOIN messages on messages_fts.id = messages.id + WHERE + messages_fts match $query + ORDER BY messages.received_at DESC, messages.sent_at DESC + LIMIT $limit; + ` + ) + .all({ + query, + limit: limit || 500, + }); - return map(rows, row => ({ - json: row.json, - snippet: row.snippet, - })); + return rows; } async function searchMessagesInConversation( @@ -2553,38 +2294,47 @@ async function searchMessagesInConversation( { limit }: { limit?: number } = {} ): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT - messages.json, - snippet(messages_fts, -1, '<>', '<>', '...', 10) as snippet - FROM messages_fts - INNER JOIN messages on messages_fts.id = messages.id - WHERE - messages_fts match $query AND - messages.conversationId = $conversationId - ORDER BY messages.received_at DESC, messages.sent_at DESC - LIMIT $limit;`, - { - $query: query, - $conversationId: conversationId, - $limit: limit || 100, - } - ); + const rows = db + .prepare( + ` + SELECT + messages.json, + snippet(messages_fts, -1, '<>', '<>', '...', 10) as snippet + FROM messages_fts + INNER JOIN messages on messages_fts.id = messages.id + WHERE + messages_fts match $query AND + messages.conversationId = $conversationId + ORDER BY messages.received_at DESC, messages.sent_at DESC + LIMIT $limit; + ` + ) + .all({ + query, + conversationId, + limit: limit || 100, + }); - return map(rows, row => ({ - json: row.json, - snippet: row.snippet, - })); + return rows; } -async function getMessageCount(conversationId?: string) { +async function getMessageCount(conversationId?: string): Promise { const db = getInstance(); - const row = conversationId - ? await db.get( - 'SELECT count(*) from messages WHERE conversationId = $conversationId;', - { $conversationId: conversationId } + let row: { 'count(*)': number } | undefined; + + if (conversationId !== undefined) { + row = db + .prepare( + ` + SELECT count(*) + FROM messages + WHERE conversationId = $conversationId; + ` ) - : await db.get('SELECT count(*) from messages;'); + .get({ conversationId }); + } else { + row = db.prepare('SELECT count(*) FROM messages;').get(); + } if (!row) { throw new Error('getMessageCount: Unable to get count of messages'); @@ -2595,12 +2345,21 @@ async function getMessageCount(conversationId?: string) { async function saveMessage( data: MessageType, - { - forceSave, - alreadyInTransaction, - }: { forceSave?: boolean; alreadyInTransaction?: boolean } = {} -) { + options: { forceSave?: boolean; alreadyInTransaction?: boolean } = {} +): Promise { const db = getInstance(); + + const { forceSave, alreadyInTransaction } = options; + + if (!alreadyInTransaction) { + return db.transaction(() => { + return saveMessage(data, { + ...options, + alreadyInTransaction: true, + }); + })(); + } + const { body, conversationId, @@ -2624,92 +2383,76 @@ async function saveMessage( } = data; const payload = { - $id: id, - $json: objectToJSON(data), + id, + json: objectToJSON(data), - $body: body, - $conversationId: conversationId, - $expirationStartTimestamp: expirationStartTimestamp, - $expires_at: expires_at, - $expireTimer: expireTimer, - $hasAttachments: hasAttachments, - $hasFileAttachments: hasFileAttachments, - $hasVisualMediaAttachments: hasVisualMediaAttachments, - $isErased: isErased, - $isViewOnce: isViewOnce, - $received_at: received_at, - $schemaVersion: schemaVersion, - $sent_at: sent_at, - $source: source, - $sourceUuid: sourceUuid, - $sourceDevice: sourceDevice, - $type: type, - $unread: unread, + body: body || null, + conversationId, + expirationStartTimestamp: expirationStartTimestamp || null, + expires_at: expires_at || null, + expireTimer: expireTimer || null, + hasAttachments: hasAttachments ? 1 : 0, + hasFileAttachments: hasFileAttachments ? 1 : 0, + hasVisualMediaAttachments: hasVisualMediaAttachments ? 1 : 0, + isErased: isErased ? 1 : 0, + isViewOnce: isViewOnce ? 1 : 0, + received_at: received_at || null, + schemaVersion, + sent_at: sent_at || null, + source: source || null, + sourceUuid: sourceUuid || null, + sourceDevice: sourceDevice || null, + type: type || null, + unread: unread ? 1 : 0, }; if (id && !forceSave) { - if (!alreadyInTransaction) { - await db.run('BEGIN TRANSACTION;'); - } + db.prepare( + ` + UPDATE messages SET + id = $id, + json = $json, - try { - await Promise.all([ - db.run( - `UPDATE messages SET - id = $id, - json = $json, + body = $body, + conversationId = $conversationId, + expirationStartTimestamp = $expirationStartTimestamp, + expires_at = $expires_at, + expireTimer = $expireTimer, + hasAttachments = $hasAttachments, + hasFileAttachments = $hasFileAttachments, + hasVisualMediaAttachments = $hasVisualMediaAttachments, + isErased = $isErased, + isViewOnce = $isViewOnce, + received_at = $received_at, + schemaVersion = $schemaVersion, + sent_at = $sent_at, + source = $source, + sourceUuid = $sourceUuid, + sourceDevice = $sourceDevice, + type = $type, + unread = $unread + WHERE id = $id; + ` + ).run(payload); + db.prepare('DELETE FROM messages_fts WHERE id = $id;').run({ + id, + }); - body = $body, - conversationId = $conversationId, - expirationStartTimestamp = $expirationStartTimestamp, - expires_at = $expires_at, - expireTimer = $expireTimer, - hasAttachments = $hasAttachments, - hasFileAttachments = $hasFileAttachments, - hasVisualMediaAttachments = $hasVisualMediaAttachments, - isErased = $isErased, - isViewOnce = $isViewOnce, - received_at = $received_at, - schemaVersion = $schemaVersion, - sent_at = $sent_at, - source = $source, - sourceUuid = $sourceUuid, - sourceDevice = $sourceDevice, - type = $type, - unread = $unread - WHERE id = $id;`, - payload - ), - db.run('DELETE FROM messages_fts WHERE id = $id;', { - $id: id, - }), - ]); - - if (body) { - await db.run( - `INSERT INTO messages_fts( - id, - body - ) VALUES ( - $id, - $body - ); - `, - { - $id: id, - $body: body, - } + if (body) { + db.prepare( + ` + INSERT INTO messages_fts( + id, + body + ) VALUES ( + $id, + $body ); - } - - if (!alreadyInTransaction) { - await db.run('COMMIT TRANSACTION;'); - } - } catch (error) { - if (!alreadyInTransaction) { - await db.run('ROLLBACK;'); - } - throw error; + ` + ).run({ + id, + body, + }); } return id; @@ -2720,191 +2463,153 @@ async function saveMessage( id: id || generateUUID(), }; - if (!alreadyInTransaction) { - await db.run('BEGIN TRANSACTION;'); - } + db.prepare('DELETE FROM messages_fts WHERE id = $id;').run({ + id, + }); - try { - await db.run('DELETE FROM messages_fts WHERE id = $id;', { - $id: id, - }); + db.prepare( + ` + INSERT INTO messages ( + id, + json, - await Promise.all([ - db.run( - `INSERT INTO messages ( - id, - json, + body, + conversationId, + expirationStartTimestamp, + expires_at, + expireTimer, + hasAttachments, + hasFileAttachments, + hasVisualMediaAttachments, + isErased, + isViewOnce, + received_at, + schemaVersion, + sent_at, + source, + sourceUuid, + sourceDevice, + type, + unread + ) values ( + $id, + $json, - body, - conversationId, - expirationStartTimestamp, - expires_at, - expireTimer, - hasAttachments, - hasFileAttachments, - hasVisualMediaAttachments, - isErased, - isViewOnce, - received_at, - schemaVersion, - sent_at, - source, - sourceUuid, - sourceDevice, - type, - unread - ) values ( - $id, - $json, - - $body, - $conversationId, - $expirationStartTimestamp, - $expires_at, - $expireTimer, - $hasAttachments, - $hasFileAttachments, - $hasVisualMediaAttachments, - $isErased, - $isViewOnce, - $received_at, - $schemaVersion, - $sent_at, - $source, - $sourceUuid, - $sourceDevice, - $type, - $unread - );`, - { - ...payload, - $id: toCreate.id, - $json: objectToJSON(toCreate), - } - ), - db.run( - `INSERT INTO messages_fts( - id, - body - ) VALUES ( - $id, - $body - ); - `, - { - $id: id, - $body: body, - } - ), - ]); - - if (!alreadyInTransaction) { - await db.run('COMMIT TRANSACTION;'); - } - } catch (error) { - if (!alreadyInTransaction) { - await db.run('ROLLBACK;'); - } - throw error; - } + $body, + $conversationId, + $expirationStartTimestamp, + $expires_at, + $expireTimer, + $hasAttachments, + $hasFileAttachments, + $hasVisualMediaAttachments, + $isErased, + $isViewOnce, + $received_at, + $schemaVersion, + $sent_at, + $source, + $sourceUuid, + $sourceDevice, + $type, + $unread + ); + ` + ).run({ + ...payload, + id: toCreate.id, + json: objectToJSON(toCreate), + }); + db.prepare( + ` + INSERT INTO messages_fts( + id, + body + ) VALUES ( + $id, + $body + ); + ` + ).run({ + id, + body, + }); return toCreate.id; } -saveMessage.needsSerial = true; async function saveMessages( arrayOfMessages: Array, { forceSave }: { forceSave?: boolean } = {} -) { +): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - ...map(arrayOfMessages, async message => - saveMessage(message, { forceSave, alreadyInTransaction: true }) - ), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + for (const message of arrayOfMessages) { + saveMessage(message, { forceSave, alreadyInTransaction: true }); + } + })(); } -saveMessages.needsSerial = true; -async function removeMessage(id: string) { +async function removeMessage(id: string): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - db.run('DELETE FROM messages WHERE id = $id;', { $id: id }), - db.run('DELETE FROM messages_fts WHERE id = $id;', { $id: id }), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + db.prepare('DELETE FROM messages WHERE id = id;').run({ id }); + db.prepare('DELETE FROM messages_fts WHERE id = id;').run({ + id, + }); + })(); } -removeMessage.needsSerial = true; -async function removeMessages(ids: Array) { +async function removeMessages(ids: Array): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - db.run( - `DELETE FROM messages WHERE id IN ( ${ids - .map(() => '?') - .join(', ')} );`, - ids - ), - db.run( - `DELETE FROM messages_fts WHERE id IN ( ${ids - .map(() => '?') - .join(', ')} );`, - ids - ), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + db.prepare( + ` + DELETE FROM messages + WHERE id IN ( ${ids.map(() => '?').join(', ')} ); + ` + ).run(ids); + db.prepare( + ` + DELETE FROM messages_fts + WHERE id IN ( ${ids.map(() => '?').join(', ')} ); + ` + ).run(ids); + })(); } -removeMessages.needsSerial = true; -async function getMessageById(id: string) { +async function getMessageById(id: string): Promise { const db = getInstance(); - const row = await db.get('SELECT * FROM messages WHERE id = $id;', { - $id: id, + const row = db.prepare('SELECT * FROM messages WHERE id = $id;').get({ + id, }); if (!row) { - return null; + return undefined; } return jsonToObject(row.json); } -async function _getAllMessages() { +async function _getAllMessages(): Promise> { const db = getInstance(); - const rows = await db.all('SELECT json FROM messages ORDER BY id ASC;'); + const rows: JSONRows = db + .prepare('SELECT json FROM messages ORDER BY id ASC;') + .all(); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function getAllMessageIds() { +async function getAllMessageIds(): Promise> { const db = getInstance(); - const rows = await db.all('SELECT id FROM messages ORDER BY id ASC;'); + const rows: Array<{ id: string }> = db + .prepare('SELECT id FROM messages ORDER BY id ASC;') + .all(); - return map(rows, row => row.id); + return rows.map(row => row.id); } async function getMessageBySender({ @@ -2917,38 +2622,46 @@ async function getMessageBySender({ sourceUuid: string; sourceDevice: string; sent_at: number; -}) { +}): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json FROM messages WHERE - (source = $source OR sourceUuid = $sourceUuid) AND - sourceDevice = $sourceDevice AND - sent_at = $sent_at;`, - { - $source: source, - $sourceUuid: sourceUuid, - $sourceDevice: sourceDevice, - $sent_at: sent_at, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages WHERE + (source = $source OR sourceUuid = $sourceUuid) AND + sourceDevice = $sourceDevice AND + sent_at = $sent_at; + ` + ) + .all({ + source, + sourceUuid, + sourceDevice, + sent_at, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function getUnreadByConversation(conversationId: string) { +async function getUnreadByConversation( + conversationId: string +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json FROM messages WHERE - unread = $unread AND - conversationId = $conversationId - ORDER BY received_at DESC, sent_at DESC;`, - { - $unread: 1, - $conversationId: conversationId, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages WHERE + unread = $unread AND + conversationId = $conversationId + ORDER BY received_at DESC, sent_at DESC; + ` + ) + .all({ + unread: 1, + conversationId, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } async function getOlderMessagesByConversation( @@ -2964,46 +2677,52 @@ async function getOlderMessagesByConversation( sentAt?: number; messageId?: string; } = {} -) { +): Promise> { const db = getInstance(); - let rows; + let rows: JSONRows; if (messageId) { - rows = await db.all( - `SELECT json FROM messages WHERE - conversationId = $conversationId AND - id != $messageId AND - ( - (received_at = $received_at AND sent_at < $sent_at) OR - received_at < $received_at - ) - ORDER BY received_at DESC, sent_at DESC - LIMIT $limit;`, - { - $conversationId: conversationId, - $received_at: receivedAt, - $sent_at: sentAt, - $limit: limit, - $messageId: messageId, - } - ); + rows = db + .prepare( + ` + SELECT json FROM messages WHERE + conversationId = $conversationId AND + id != $messageId AND + ( + (received_at = $received_at AND sent_at < $sent_at) OR + received_at < $received_at + ) + ORDER BY received_at DESC, sent_at DESC + LIMIT $limit; + ` + ) + .all({ + conversationId, + received_at: receivedAt, + sent_at: sentAt, + limit, + messageId, + }); } else { - rows = await db.all( - `SELECT json FROM messages WHERE - conversationId = $conversationId AND - ( - (received_at = $received_at AND sent_at < $sent_at) OR - received_at < $received_at - ) - ORDER BY received_at DESC, sent_at DESC - LIMIT $limit;`, - { - $conversationId: conversationId, - $received_at: receivedAt, - $sent_at: sentAt, - $limit: limit, - } - ); + rows = db + .prepare( + ` + SELECT json FROM messages WHERE + conversationId = $conversationId AND + ( + (received_at = $received_at AND sent_at < $sent_at) OR + received_at < $received_at + ) + ORDER BY received_at DESC, sent_at DESC + LIMIT $limit; + ` + ) + .all({ + conversationId, + received_at: receivedAt, + sent_at: sentAt, + limit, + }); } return rows.reverse(); @@ -3016,59 +2735,72 @@ async function getNewerMessagesByConversation( receivedAt = 0, sentAt = 0, }: { limit?: number; receivedAt?: number; sentAt?: number } = {} -) { +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json FROM messages WHERE - conversationId = $conversationId AND - ( - (received_at = $received_at AND sent_at > $sent_at) OR - received_at > $received_at - ) - ORDER BY received_at ASC, sent_at ASC - LIMIT $limit;`, - { - $conversationId: conversationId, - $received_at: receivedAt, - $sent_at: sentAt, - $limit: limit, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages WHERE + conversationId = $conversationId AND + ( + (received_at = $received_at AND sent_at > $sent_at) OR + received_at > $received_at + ) + ORDER BY received_at ASC, sent_at ASC + LIMIT $limit; + ` + ) + .all({ + conversationId, + received_at: receivedAt, + sent_at: sentAt, + limit, + }); return rows; } -async function getOldestMessageForConversation(conversationId: string) { +function getOldestMessageForConversation( + conversationId: string +): MessageMetricsType | undefined { const db = getInstance(); - const row = await db.get( - `SELECT * FROM messages WHERE - conversationId = $conversationId - ORDER BY received_at ASC, sent_at ASC - LIMIT 1;`, - { - $conversationId: conversationId, - } - ); + const row = db + .prepare( + ` + SELECT * FROM messages WHERE + conversationId = $conversationId + ORDER BY received_at ASC, sent_at ASC + LIMIT 1; + ` + ) + .get({ + conversationId, + }); if (!row) { - return null; + return undefined; } return row; } -async function getNewestMessageForConversation(conversationId: string) { +function getNewestMessageForConversation( + conversationId: string +): MessageMetricsType | undefined { const db = getInstance(); - const row = await db.get( - `SELECT * FROM messages WHERE - conversationId = $conversationId - ORDER BY received_at DESC, sent_at DESC - LIMIT 1;`, - { - $conversationId: conversationId, - } - ); + const row = db + .prepare( + ` + SELECT * FROM messages WHERE + conversationId = $conversationId + ORDER BY received_at DESC, sent_at DESC + LIMIT 1; + ` + ) + .get({ + conversationId, + }); if (!row) { - return null; + return undefined; } return row; @@ -3080,43 +2812,47 @@ async function getLastConversationActivity({ }: { conversationId: string; ourConversationId: string; -}): Promise { +}): Promise { const db = getInstance(); - const row = await db.get( - `SELECT * FROM messages WHERE - conversationId = $conversationId AND - (type IS NULL - OR - type NOT IN ( - 'profile-change', - 'verified-change', - 'message-history-unsynced', - 'keychange', - 'group-v1-migration' + const row = db + .prepare( + ` + SELECT * FROM messages + WHERE + conversationId = $conversationId AND + (type IS NULL + OR + type NOT IN ( + 'profile-change', + 'verified-change', + 'message-history-unsynced', + 'keychange', + 'group-v1-migration' + ) + ) AND + ( + json_extract(json, '$.expirationTimerUpdate.fromSync') IS NULL + OR + json_extract(json, '$.expirationTimerUpdate.fromSync') != 1 + ) AND NOT + ( + type = 'group-v2-change' AND + json_extract(json, '$.groupV2Change.from') != $ourConversationId AND + json_extract(json, '$.groupV2Change.details.length') = 1 AND + json_extract(json, '$.groupV2Change.details[0].type') = 'member-remove' AND + json_extract(json, '$.groupV2Change.details[0].conversationId') != $ourConversationId ) - ) AND - ( - json_extract(json, '$.expirationTimerUpdate.fromSync') IS NULL - OR - json_extract(json, '$.expirationTimerUpdate.fromSync') != 1 - ) AND NOT - ( - type = 'group-v2-change' AND - json_extract(json, '$.groupV2Change.from') != $ourConversationId AND - json_extract(json, '$.groupV2Change.details.length') = 1 AND - json_extract(json, '$.groupV2Change.details[0].type') = 'member-remove' AND - json_extract(json, '$.groupV2Change.details[0].conversationId') != $ourConversationId - ) - ORDER BY received_at DESC, sent_at DESC - LIMIT 1;`, - { - $conversationId: conversationId, - $ourConversationId: ourConversationId, - } - ); + ORDER BY received_at DESC, sent_at DESC + LIMIT 1; + ` + ) + .get({ + conversationId, + ourConversationId, + }); if (!row) { - return null; + return undefined; } return jsonToObject(row.json); @@ -3127,73 +2863,86 @@ async function getLastConversationPreview({ }: { conversationId: string; ourConversationId: string; -}): Promise { +}): Promise { const db = getInstance(); - const row = await db.get( - `SELECT * FROM messages WHERE - conversationId = $conversationId AND - ( - type IS NULL - OR - type NOT IN ( - 'profile-change', - 'verified-change', - 'message-history-unsynced', - 'group-v1-migration' + const row = db + .prepare( + ` + SELECT * FROM messages + WHERE + conversationId = $conversationId AND + ( + type IS NULL + OR + type NOT IN ( + 'profile-change', + 'verified-change', + 'message-history-unsynced', + 'group-v1-migration' + ) + ) AND NOT + ( + type = 'group-v2-change' AND + json_extract(json, '$.groupV2Change.from') != $ourConversationId AND + json_extract(json, '$.groupV2Change.details.length') = 1 AND + json_extract(json, '$.groupV2Change.details[0].type') = 'member-remove' AND + json_extract(json, '$.groupV2Change.details[0].conversationId') != $ourConversationId ) - ) AND NOT - ( - type = 'group-v2-change' AND - json_extract(json, '$.groupV2Change.from') != $ourConversationId AND - json_extract(json, '$.groupV2Change.details.length') = 1 AND - json_extract(json, '$.groupV2Change.details[0].type') = 'member-remove' AND - json_extract(json, '$.groupV2Change.details[0].conversationId') != $ourConversationId - ) - ORDER BY received_at DESC, sent_at DESC - LIMIT 1;`, - { - $conversationId: conversationId, - $ourConversationId: ourConversationId, - } - ); + ORDER BY received_at DESC, sent_at DESC + LIMIT 1; + ` + ) + .get({ + conversationId, + ourConversationId, + }); if (!row) { - return null; + return undefined; } return jsonToObject(row.json); } -async function getOldestUnreadMessageForConversation(conversationId: string) { +function getOldestUnreadMessageForConversation( + conversationId: string +): MessageMetricsType | undefined { const db = getInstance(); - const row = await db.get( - `SELECT * FROM messages WHERE - conversationId = $conversationId AND - unread = 1 - ORDER BY received_at ASC, sent_at ASC - LIMIT 1;`, - { - $conversationId: conversationId, - } - ); + const row = db + .prepare( + ` + SELECT * FROM messages WHERE + conversationId = $conversationId AND + unread = 1 + ORDER BY received_at ASC, sent_at ASC + LIMIT 1; + ` + ) + .get({ + conversationId, + }); if (!row) { - return null; + return undefined; } return row; } -async function getTotalUnreadForConversation(conversationId: string) { +function getTotalUnreadForConversation(conversationId: string): number { const db = getInstance(); - const row = await db.get( - `SELECT count(id) from messages WHERE - conversationId = $conversationId AND - unread = 1; - `, - { - $conversationId: conversationId, - } - ); + const row = db + .prepare( + ` + SELECT count(id) + FROM messages + WHERE + conversationId = $conversationId AND + unread = 1; + ` + ) + .get({ + conversationId, + }); if (!row) { throw new Error('getTotalUnreadForConversation: Unable to get count'); @@ -3202,26 +2951,23 @@ async function getTotalUnreadForConversation(conversationId: string) { return row['count(id)']; } -async function getMessageMetricsForConversation(conversationId: string) { - const results = await Promise.all([ - getOldestMessageForConversation(conversationId), - getNewestMessageForConversation(conversationId), - getOldestUnreadMessageForConversation(conversationId), - getTotalUnreadForConversation(conversationId), - ]); - - const [oldest, newest, oldestUnread, totalUnread] = results; +async function getMessageMetricsForConversation( + conversationId: string +): Promise { + const oldest = getOldestMessageForConversation(conversationId); + const newest = getNewestMessageForConversation(conversationId); + const oldestUnread = getOldestUnreadMessageForConversation(conversationId); + const totalUnread = getTotalUnreadForConversation(conversationId); return { - oldest: oldest ? pick(oldest, ['received_at', 'sent_at', 'id']) : null, - newest: newest ? pick(newest, ['received_at', 'sent_at', 'id']) : null, + oldest: oldest ? pick(oldest, ['received_at', 'sent_at', 'id']) : undefined, + newest: newest ? pick(newest, ['received_at', 'sent_at', 'id']) : undefined, oldestUnread: oldestUnread ? pick(oldestUnread, ['received_at', 'sent_at', 'id']) - : null, + : undefined, totalUnread, }; } -getMessageMetricsForConversation.needsSerial = true; async function hasGroupCallHistoryMessage( conversationId: string, @@ -3229,24 +2975,24 @@ async function hasGroupCallHistoryMessage( ): Promise { const db = getInstance(); - const row: unknown = await db.get( - ` - SELECT count(*) FROM messages - WHERE conversationId = $conversationId - AND type = 'call-history' - AND json_extract(json, '$.callHistoryDetails.callMode') = 'Group' - AND json_extract(json, '$.callHistoryDetails.eraId') = $eraId - LIMIT 1; - `, - { - $conversationId: conversationId, - $eraId: eraId, - } - ); + const row: { 'count(*)': number } | undefined = db + .prepare( + ` + SELECT count(*) FROM messages + WHERE conversationId = $conversationId + AND type = 'call-history' + AND json_extract(json, '$.callHistoryDetails.callMode') = 'Group' + AND json_extract(json, '$.callHistoryDetails.eraId') = $eraId + LIMIT 1; + ` + ) + .get({ + conversationId, + eraId, + }); - if (typeof row === 'object' && row && !Array.isArray(row)) { - const count = Number((row as Record)['count(*)']); - return Boolean(count); + if (row) { + return Boolean(row['count(*)']); } return false; } @@ -3254,127 +3000,153 @@ async function hasGroupCallHistoryMessage( async function migrateConversationMessages( obsoleteId: string, currentId: string -) { +): Promise { const db = getInstance(); - await db.run( - `UPDATE messages SET + db.prepare( + ` + UPDATE messages SET conversationId = $currentId, json = json_set(json, '$.conversationId', $currentId) - WHERE conversationId = $obsoleteId;`, - { - $obsoleteId: obsoleteId, - $currentId: currentId, - } - ); + WHERE conversationId = $obsoleteId; + ` + ).run({ + obsoleteId, + currentId, + }); } -migrateConversationMessages.needsSerial = true; -async function getMessagesBySentAt(sentAt: number) { +async function getMessagesBySentAt( + sentAt: number +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT * FROM messages - WHERE sent_at = $sent_at - ORDER BY received_at DESC, sent_at DESC;`, - { - $sent_at: sentAt, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages + WHERE sent_at = $sent_at + ORDER BY received_at DESC, sent_at DESC; + ` + ) + .all({ + sent_at: sentAt, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function getExpiredMessages() { +async function getExpiredMessages(): Promise> { const db = getInstance(); const now = Date.now(); - const rows = await db.all( - `SELECT json FROM messages WHERE - expires_at IS NOT NULL AND - expires_at <= $expires_at - ORDER BY expires_at ASC;`, - { - $expires_at: now, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages WHERE + expires_at IS NOT NULL AND + expires_at <= $expires_at + ORDER BY expires_at ASC; + ` + ) + .all({ + expires_at: now, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function getOutgoingWithoutExpiresAt() { +async function getOutgoingWithoutExpiresAt(): Promise> { const db = getInstance(); - const rows = await db.all(` - SELECT json FROM messages - INDEXED BY messages_without_timer - WHERE - expireTimer > 0 AND - expires_at IS NULL AND - type IS 'outgoing' - ORDER BY expires_at ASC; - `); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages + INDEXED BY messages_without_timer + WHERE + expireTimer > 0 AND + expires_at IS NULL AND + type IS 'outgoing' + ORDER BY expires_at ASC; + ` + ) + .all(); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function getNextExpiringMessage() { +async function getNextExpiringMessage(): Promise { const db = getInstance(); // Note: we avoid 'IS NOT NULL' here because it does seem to bypass our index - const rows = await db.all(` - SELECT json FROM messages - WHERE expires_at > 0 - ORDER BY expires_at ASC - LIMIT 1; - `); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages + WHERE expires_at > 0 + ORDER BY expires_at ASC + LIMIT 1; + ` + ) + .all(); if (!rows || rows.length < 1) { - return null; + return undefined; } return jsonToObject(rows[0].json); } -async function getNextTapToViewMessageToAgeOut() { +async function getNextTapToViewMessageToAgeOut(): Promise< + MessageType | undefined +> { const db = getInstance(); - const rows = await db.all(` - SELECT json FROM messages - WHERE - isViewOnce = 1 - AND (isErased IS NULL OR isErased != 1) - ORDER BY received_at ASC, sent_at ASC - LIMIT 1; - `); + const rows = db + .prepare( + ` + SELECT json FROM messages + WHERE + isViewOnce = 1 + AND (isErased IS NULL OR isErased != 1) + ORDER BY received_at ASC, sent_at ASC + LIMIT 1; + ` + ) + .all(); if (!rows || rows.length < 1) { - return null; + return undefined; } return jsonToObject(rows[0].json); } -async function getTapToViewMessagesNeedingErase() { +async function getTapToViewMessagesNeedingErase(): Promise> { const db = getInstance(); const THIRTY_DAYS_AGO = Date.now() - 30 * 24 * 60 * 60 * 1000; - const rows = await db.all( - `SELECT json FROM messages - WHERE - isViewOnce = 1 - AND (isErased IS NULL OR isErased != 1) - AND received_at <= $THIRTY_DAYS_AGO - ORDER BY received_at ASC, sent_at ASC;`, - { - $THIRTY_DAYS_AGO: THIRTY_DAYS_AGO, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json + FROM messages + WHERE + isViewOnce = 1 + AND (isErased IS NULL OR isErased != 1) + AND received_at <= $THIRTY_DAYS_AGO + ORDER BY received_at ASC, sent_at ASC; + ` + ) + .all({ + THIRTY_DAYS_AGO, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } async function saveUnprocessed( data: UnprocessedType, { forceSave }: { forceSave?: boolean } = {} -) { +): Promise { const db = getInstance(); const { id, timestamp, version, attempts, envelope } = data; if (!id) { @@ -3382,8 +3154,9 @@ async function saveUnprocessed( } if (forceSave) { - await db.run( - `INSERT INTO unprocessed ( + db.prepare( + ` + INSERT INTO unprocessed ( id, timestamp, version, @@ -3395,34 +3168,35 @@ async function saveUnprocessed( $version, $attempts, $envelope - );`, - { - $id: id, - $timestamp: timestamp, - $version: version, - $attempts: attempts, - $envelope: envelope, - } - ); + ); + ` + ).run({ + id, + timestamp, + version, + attempts, + envelope, + }); return id; } - await db.run( - `UPDATE unprocessed SET + db.prepare( + ` + UPDATE unprocessed SET timestamp = $timestamp, version = $version, attempts = $attempts, envelope = $envelope - WHERE id = $id;`, - { - $id: id, - $timestamp: timestamp, - $version: version, - $attempts: attempts, - $envelope: envelope, - } - ); + WHERE id = $id; + ` + ).run({ + id, + timestamp, + version, + attempts, + envelope, + }); return id; } @@ -3430,87 +3204,86 @@ async function saveUnprocessed( async function saveUnprocesseds( arrayOfUnprocessed: Array, { forceSave }: { forceSave?: boolean } = {} -) { +): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - ...map(arrayOfUnprocessed, async unprocessed => - saveUnprocessed(unprocessed, { forceSave }) - ), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + for (const unprocessed of arrayOfUnprocessed) { + saveUnprocessed(unprocessed, { forceSave }); + } + })(); } -saveUnprocesseds.needsSerial = true; -async function updateUnprocessedAttempts(id: string, attempts: number) { +async function updateUnprocessedAttempts( + id: string, + attempts: number +): Promise { const db = getInstance(); - await db.run('UPDATE unprocessed SET attempts = $attempts WHERE id = $id;', { - $id: id, - $attempts: attempts, + db.prepare( + ` + UPDATE unprocessed + SET attempts = $attempts + WHERE id = $id; + ` + ).run({ + id, + attempts, }); } -async function updateUnprocessedWithData(id: string, data: UnprocessedType) { +async function updateUnprocessedWithData( + id: string, + data: UnprocessedType +): Promise { const db = getInstance(); const { source, sourceUuid, sourceDevice, serverTimestamp, decrypted } = data; - await db.run( - `UPDATE unprocessed SET + db.prepare( + ` + UPDATE unprocessed SET source = $source, sourceUuid = $sourceUuid, sourceDevice = $sourceDevice, serverTimestamp = $serverTimestamp, decrypted = $decrypted - WHERE id = $id;`, - { - $id: id, - $source: source, - $sourceUuid: sourceUuid, - $sourceDevice: sourceDevice, - $serverTimestamp: serverTimestamp, - $decrypted: decrypted, - } - ); + WHERE id = $id; + ` + ).run({ + id, + source: source || null, + sourceUuid: sourceUuid || null, + sourceDevice: sourceDevice || null, + serverTimestamp: serverTimestamp || null, + decrypted: decrypted || null, + }); } async function updateUnprocessedsWithData( - arrayOfUnprocessed: Array -) { + arrayOfUnprocessed: Array<{ id: string; data: UnprocessedType }> +): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - ...map(arrayOfUnprocessed, async ({ id, data }) => - updateUnprocessedWithData(id, data) - ), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + for (const { id, data } of arrayOfUnprocessed) { + updateUnprocessedWithData(id, data); + } + })(); } -updateUnprocessedsWithData.needsSerial = true; -async function getUnprocessedById(id: string) { +async function getUnprocessedById( + id: string +): Promise { const db = getInstance(); - const row = await db.get('SELECT * FROM unprocessed WHERE id = $id;', { - $id: id, - }); + const row = db + .prepare('SELECT * FROM unprocessed WHERE id = $id;') + .get({ + id, + }); return row; } -async function getUnprocessedCount() { +async function getUnprocessedCount(): Promise { const db = getInstance(); - const row = await db.get('SELECT count(*) from unprocessed;'); + const row = db.prepare('SELECT count(*) from unprocessed;').get(); if (!row) { throw new Error('getMessageCount: Unable to get count of unprocessed'); @@ -3519,20 +3292,26 @@ async function getUnprocessedCount() { return row['count(*)']; } -async function getAllUnprocessed() { +async function getAllUnprocessed(): Promise> { const db = getInstance(); - const rows = await db.all( - 'SELECT * FROM unprocessed ORDER BY timestamp ASC;' - ); + const rows = db + .prepare( + ` + SELECT * + FROM unprocessed + ORDER BY timestamp ASC; + ` + ) + .all(); return rows; } -async function removeUnprocessed(id: string | Array) { +async function removeUnprocessed(id: string | Array): Promise { const db = getInstance(); if (!Array.isArray(id)) { - await db.run('DELETE FROM unprocessed WHERE id = $id;', { $id: id }); + db.prepare('DELETE FROM unprocessed WHERE id = id;').run({ id }); return; } @@ -3542,15 +3321,17 @@ async function removeUnprocessed(id: string | Array) { } // Our node interface doesn't seem to allow you to replace one single ? with an array - await db.run( - `DELETE FROM unprocessed WHERE id IN ( ${id.map(() => '?').join(', ')} );`, - id - ); + db.prepare( + ` + DELETE FROM unprocessed + WHERE id IN ( ${id.map(() => '?').join(', ')} ); + ` + ).run(id); } -async function removeAllUnprocessed() { +async function removeAllUnprocessed(): Promise { const db = getInstance(); - await db.run('DELETE FROM unprocessed;'); + db.prepare('DELETE FROM unprocessed;').run(); } // Attachment Downloads @@ -3559,25 +3340,31 @@ const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; async function getNextAttachmentDownloadJobs( limit?: number, options: { timestamp?: number } = {} -) { +): Promise> { const db = getInstance(); const timestamp = options && options.timestamp ? options.timestamp : Date.now(); - const rows = await db.all( - `SELECT json FROM attachment_downloads - WHERE pending = 0 AND timestamp < $timestamp - ORDER BY timestamp DESC - LIMIT $limit;`, - { - $limit: limit || 3, - $timestamp: timestamp, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json + FROM attachment_downloads + WHERE pending = 0 AND timestamp < $timestamp + ORDER BY timestamp DESC + LIMIT $limit; + ` + ) + .all({ + limit: limit || 3, + timestamp, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } -async function saveAttachmentDownloadJob(job: AttachmentDownloadJobType) { +async function saveAttachmentDownloadJob( + job: AttachmentDownloadJobType +): Promise { const db = getInstance(); const { id, pending, timestamp } = job; if (!id) { @@ -3586,8 +3373,9 @@ async function saveAttachmentDownloadJob(job: AttachmentDownloadJobType) { ); } - await db.run( - `INSERT OR REPLACE INTO attachment_downloads ( + db.prepare( + ` + INSERT OR REPLACE INTO attachment_downloads ( id, pending, timestamp, @@ -3597,41 +3385,51 @@ async function saveAttachmentDownloadJob(job: AttachmentDownloadJobType) { $pending, $timestamp, $json - )`, - { - $id: id, - $pending: pending, - $timestamp: timestamp, - $json: objectToJSON(job), - } - ); + ) + ` + ).run({ + id, + pending, + timestamp, + json: objectToJSON(job), + }); } -async function setAttachmentDownloadJobPending(id: string, pending: boolean) { +async function setAttachmentDownloadJobPending( + id: string, + pending: boolean +): Promise { const db = getInstance(); - await db.run( - 'UPDATE attachment_downloads SET pending = $pending WHERE id = $id;', - { - $id: id, - $pending: pending, - } - ); + db.prepare( + ` + UPDATE attachment_downloads + SET pending = $pending + WHERE id = $id; + ` + ).run({ + id, + pending: pending ? 1 : 0, + }); } -async function resetAttachmentDownloadPending() { +async function resetAttachmentDownloadPending(): Promise { const db = getInstance(); - await db.run( - 'UPDATE attachment_downloads SET pending = 0 WHERE pending != 0;' - ); + db.prepare( + ` + UPDATE attachment_downloads + SET pending = 0 + WHERE pending != 0; + ` + ).run(); } -async function removeAttachmentDownloadJob(id: string) { +function removeAttachmentDownloadJob(id: string): Promise { return removeById(ATTACHMENT_DOWNLOADS_TABLE, id); } -async function removeAllAttachmentDownloadJobs() { +function removeAllAttachmentDownloadJobs(): Promise { return removeAllFromTable(ATTACHMENT_DOWNLOADS_TABLE); } // Stickers -async function createOrUpdateStickerPack(pack: StickerPackType) { +async function createOrUpdateStickerPack(pack: StickerPackType): Promise { const db = getInstance(); const { attemptedStatus, @@ -3653,27 +3451,34 @@ async function createOrUpdateStickerPack(pack: StickerPackType) { ); } - const rows = await db.all('SELECT id FROM sticker_packs WHERE id = $id;', { - $id: id, - }); + const rows = db + .prepare( + ` + SELECT id + FROM sticker_packs + WHERE id = $id; + ` + ) + .all({ id }); const payload = { - $attemptedStatus: attemptedStatus, - $author: author, - $coverStickerId: coverStickerId, - $createdAt: createdAt || Date.now(), - $downloadAttempts: downloadAttempts || 1, - $id: id, - $installedAt: installedAt, - $key: key, - $lastUsed: lastUsed || null, - $status: status, - $stickerCount: stickerCount, - $title: title, + attemptedStatus, + author, + coverStickerId, + createdAt: createdAt || Date.now(), + downloadAttempts: downloadAttempts || 1, + id, + installedAt, + key, + lastUsed: lastUsed || null, + status, + stickerCount, + title, }; if (rows && rows.length) { - await db.run( - `UPDATE sticker_packs SET + db.prepare( + ` + UPDATE sticker_packs SET attemptedStatus = $attemptedStatus, author = $author, coverStickerId = $coverStickerId, @@ -3685,15 +3490,16 @@ async function createOrUpdateStickerPack(pack: StickerPackType) { status = $status, stickerCount = $stickerCount, title = $title - WHERE id = $id;`, - payload - ); + WHERE id = $id; + ` + ).run(payload); return; } - await db.run( - `INSERT INTO sticker_packs ( + db.prepare( + ` + INSERT INTO sticker_packs ( attemptedStatus, author, coverStickerId, @@ -3719,39 +3525,44 @@ async function createOrUpdateStickerPack(pack: StickerPackType) { $status, $stickerCount, $title - )`, - payload - ); + ) + ` + ).run(payload); } async function updateStickerPackStatus( id: string, status: StickerPackStatusType, options?: { timestamp: number } -) { +): Promise { const db = getInstance(); const timestamp = options ? options.timestamp || Date.now() : Date.now(); const installedAt = status === 'installed' ? timestamp : null; - await db.run( - `UPDATE sticker_packs + db.prepare( + ` + UPDATE sticker_packs SET status = $status, installedAt = $installedAt WHERE id = $id; - )`, - { - $id: id, - $status: status, - $installedAt: installedAt, - } - ); + ) + ` + ).run({ + id, + status, + installedAt, + }); } async function clearAllErrorStickerPackAttempts(): Promise { const db = getInstance(); - await db.run( - "UPDATE sticker_packs SET downloadAttempts = 0 WHERE status = 'error';" - ); + db.prepare( + ` + UPDATE sticker_packs + SET downloadAttempts = 0 + WHERE status = 'error'; + ` + ).run(); } -async function createOrUpdateSticker(sticker: StickerType) { +async function createOrUpdateSticker(sticker: StickerType): Promise { const db = getInstance(); const { emoji, @@ -3775,8 +3586,9 @@ async function createOrUpdateSticker(sticker: StickerType) { ); } - await db.run( - `INSERT OR REPLACE INTO stickers ( + db.prepare( + ` + INSERT OR REPLACE INTO stickers ( emoji, height, id, @@ -3794,46 +3606,51 @@ async function createOrUpdateSticker(sticker: StickerType) { $packId, $path, $width - )`, - { - $emoji: emoji, - $height: height, - $id: id, - $isCoverOnly: isCoverOnly, - $lastUsed: lastUsed, - $packId: packId, - $path: path, - $width: width, - } - ); + ) + ` + ).run({ + emoji, + height, + id, + isCoverOnly, + lastUsed, + packId, + path, + width, + }); } async function updateStickerLastUsed( packId: string, stickerId: number, lastUsed: number -) { +): Promise { const db = getInstance(); - await db.run( - `UPDATE stickers + db.prepare( + ` + UPDATE stickers SET lastUsed = $lastUsed - WHERE id = $id AND packId = $packId;`, - { - $id: stickerId, - $packId: packId, - $lastUsed: lastUsed, - } - ); - await db.run( - `UPDATE sticker_packs + WHERE id = $id AND packId = $packId; + ` + ).run({ + id: stickerId, + packId, + lastUsed, + }); + db.prepare( + ` + UPDATE sticker_packs SET lastUsed = $lastUsed - WHERE id = $id;`, - { - $id: packId, - $lastUsed: lastUsed, - } - ); + WHERE id = $id; + ` + ).run({ + id: packId, + lastUsed, + }); } -async function addStickerPackReference(messageId: string, packId: string) { +async function addStickerPackReference( + messageId: string, + packId: string +): Promise { const db = getInstance(); if (!messageId) { @@ -3847,21 +3664,25 @@ async function addStickerPackReference(messageId: string, packId: string) { ); } - await db.run( - `INSERT OR REPLACE INTO sticker_references ( + db.prepare( + ` + INSERT OR REPLACE INTO sticker_references ( messageId, packId ) values ( $messageId, $packId - )`, - { - $messageId: messageId, - $packId: packId, - } - ); + ) + ` + ).all({ + messageId, + packId, + }); } -async function deleteStickerPackReference(messageId: string, packId: string) { +async function deleteStickerPackReference( + messageId: string, + packId: string +): Promise> { const db = getInstance(); if (!messageId) { @@ -3875,89 +3696,90 @@ async function deleteStickerPackReference(messageId: string, packId: string) { ); } - try { - // We use an immediate transaction here to immediately acquire an exclusive lock, - // which would normally only happen when we did our first write. + return db + .transaction(() => { + // We use an immediate transaction here to immediately acquire an exclusive lock, + // which would normally only happen when we did our first write. - // We need this to ensure that our five queries are all atomic, with no other changes - // happening while we do it: - // 1. Delete our target messageId/packId references - // 2. Check the number of references still pointing at packId - // 3. If that number is zero, get pack from sticker_packs database - // 4. If it's not installed, then grab all of its sticker paths - // 5. If it's not installed, then sticker pack (which cascades to all stickers and - // references) - await db.run('BEGIN IMMEDIATE TRANSACTION;'); + // We need this to ensure that our five queries are all atomic, with no + // other changes happening while we do it: + // 1. Delete our target messageId/packId references + // 2. Check the number of references still pointing at packId + // 3. If that number is zero, get pack from sticker_packs database + // 4. If it's not installed, then grab all of its sticker paths + // 5. If it's not installed, then sticker pack (which cascades to all + // stickers and references) + db.prepare( + ` + DELETE FROM sticker_references + WHERE messageId = $messageId AND packId = $packId; + ` + ).run({ + messageId, + packId, + }); - await db.run( - `DELETE FROM sticker_references - WHERE messageId = $messageId AND packId = $packId;`, - { - $messageId: messageId, - $packId: packId, + const countRow = db + .prepare( + ` + SELECT count(*) FROM sticker_references + WHERE packId = $packId; + ` + ) + .get({ packId }); + if (!countRow) { + throw new Error( + 'deleteStickerPackReference: Unable to get count of references' + ); } - ); - - const countRow = await db.get( - `SELECT count(*) FROM sticker_references - WHERE packId = $packId;`, - { $packId: packId } - ); - if (!countRow) { - throw new Error( - 'deleteStickerPackReference: Unable to get count of references' - ); - } - const count = countRow['count(*)']; - if (count > 0) { - await db.run('COMMIT TRANSACTION;'); - - return []; - } - - const packRow = await db.get( - `SELECT status FROM sticker_packs - WHERE id = $packId;`, - { $packId: packId } - ); - if (!packRow) { - console.log('deleteStickerPackReference: did not find referenced pack'); - await db.run('COMMIT TRANSACTION;'); - - return []; - } - const { status } = packRow; - - if (status === 'installed') { - await db.run('COMMIT TRANSACTION;'); - - return []; - } - - const stickerPathRows = await db.all( - `SELECT path FROM stickers - WHERE packId = $packId;`, - { - $packId: packId, + const count = countRow['count(*)']; + if (count > 0) { + return []; } - ); - await db.run( - `DELETE FROM sticker_packs - WHERE id = $packId;`, - { $packId: packId } - ); - await db.run('COMMIT TRANSACTION;'); + const packRow: { status: StickerPackStatusType } = db + .prepare( + ` + SELECT status FROM sticker_packs + WHERE id = $packId; + ` + ) + .get({ packId }); + if (!packRow) { + console.log('deleteStickerPackReference: did not find referenced pack'); + return []; + } + const { status } = packRow; - return (stickerPathRows || []).map(row => row.path); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + if (status === 'installed') { + return []; + } + + const stickerPathRows: Array<{ path: string }> = db + .prepare( + ` + SELECT path FROM stickers + WHERE packId = $packId; + ` + ) + .all({ + packId, + }); + db.prepare( + ` + DELETE FROM sticker_packs + WHERE id = $packId; + ` + ).run({ + packId, + }); + + return (stickerPathRows || []).map(row => row.path); + }) + .immediate(); } -deleteStickerPackReference.needsSerial = true; -async function deleteStickerPack(packId: string) { +async function deleteStickerPack(packId: string): Promise> { const db = getInstance(); if (!packId) { @@ -3966,43 +3788,42 @@ async function deleteStickerPack(packId: string) { ); } - try { - // We use an immediate transaction here to immediately acquire an exclusive lock, - // which would normally only happen when we did our first write. + return db + .transaction(() => { + // We use an immediate transaction here to immediately acquire an exclusive lock, + // which would normally only happen when we did our first write. - // We need this to ensure that our two queries are atomic, with no other changes - // happening while we do it: - // 1. Grab all of target pack's sticker paths - // 2. Delete sticker pack (which cascades to all stickers and references) - await db.run('BEGIN IMMEDIATE TRANSACTION;'); + // We need this to ensure that our two queries are atomic, with no other changes + // happening while we do it: + // 1. Grab all of target pack's sticker paths + // 2. Delete sticker pack (which cascades to all stickers and references) - const stickerPathRows = await db.all( - `SELECT path FROM stickers - WHERE packId = $packId;`, - { - $packId: packId, - } - ); - await db.run( - `DELETE FROM sticker_packs - WHERE id = $packId;`, - { $packId: packId } - ); + const stickerPathRows: Array<{ path: string }> = db + .prepare( + ` + SELECT path FROM stickers + WHERE packId = $packId; + ` + ) + .all({ + packId, + }); + db.prepare( + ` + DELETE FROM sticker_packs + WHERE id = $packId; + ` + ).run({ packId }); - await db.run('COMMIT TRANSACTION;'); - - return (stickerPathRows || []).map(row => row.path); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + return (stickerPathRows || []).map(row => row.path); + }) + .immediate(); } -deleteStickerPack.needsSerial = true; -async function getStickerCount() { +async function getStickerCount(): Promise { const db = getInstance(); - const row = await db.get('SELECT count(*) from stickers;'); + const row = db.prepare('SELECT count(*) from stickers;').get(); if (!row) { throw new Error('getStickerCount: Unable to get count of stickers'); @@ -4010,40 +3831,53 @@ async function getStickerCount() { return row['count(*)']; } -async function getAllStickerPacks() { +async function getAllStickerPacks(): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT * FROM sticker_packs - ORDER BY installedAt DESC, createdAt DESC` - ); + const rows = db + .prepare( + ` + SELECT * FROM sticker_packs + ORDER BY installedAt DESC, createdAt DESC + ` + ) + .all(); return rows || []; } -async function getAllStickers() { +async function getAllStickers(): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT * FROM stickers - ORDER BY packId ASC, id ASC` - ); + const rows = db + .prepare( + ` + SELECT * FROM stickers + ORDER BY packId ASC, id ASC + ` + ) + .all(); return rows || []; } -async function getRecentStickers({ limit }: { limit?: number } = {}) { +async function getRecentStickers({ limit }: { limit?: number } = {}): Promise< + Array +> { const db = getInstance(); // Note: we avoid 'IS NOT NULL' here because it does seem to bypass our index - const rows = await db.all( - `SELECT stickers.* FROM stickers - JOIN sticker_packs on stickers.packId = sticker_packs.id - WHERE stickers.lastUsed > 0 AND sticker_packs.status = 'installed' - ORDER BY stickers.lastUsed DESC - LIMIT $limit`, - { - $limit: limit || 24, - } - ); + const rows = db + .prepare( + ` + SELECT stickers.* FROM stickers + JOIN sticker_packs on stickers.packId = sticker_packs.id + WHERE stickers.lastUsed > 0 AND sticker_packs.status = 'installed' + ORDER BY stickers.lastUsed DESC + LIMIT $limit + ` + ) + .all({ + limit: limit || 24, + }); return rows || []; } @@ -4052,162 +3886,164 @@ async function getRecentStickers({ limit }: { limit?: number } = {}) { async function updateEmojiUsage( shortName: string, timeUsed: number = Date.now() -) { +): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - const rows = await db.get( - 'SELECT * FROM emojis WHERE shortName = $shortName;', - { - $shortName: shortName, - } - ); + db.transaction(() => { + const rows = db + .prepare( + ` + SELECT * FROM emojis + WHERE shortName = $shortName; + ` + ) + .get({ + shortName, + }); if (rows) { - await db.run( - 'UPDATE emojis SET lastUsage = $timeUsed WHERE shortName = $shortName;', - { $shortName: shortName, $timeUsed: timeUsed } - ); + db.prepare( + ` + UPDATE emojis + SET lastUsage = $timeUsed + WHERE shortName = $shortName; + ` + ).run({ shortName, timeUsed }); } else { - await db.run( - 'INSERT INTO emojis(shortName, lastUsage) VALUES ($shortName, $timeUsed);', - { $shortName: shortName, $timeUsed: timeUsed } - ); + db.prepare( + ` + INSERT INTO emojis(shortName, lastUsage) + VALUES ($shortName, $timeUsed); + ` + ).run({ shortName, timeUsed }); } - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + })(); } -updateEmojiUsage.needsSerial = true; -async function getRecentEmojis(limit = 32) { +async function getRecentEmojis(limit = 32): Promise> { const db = getInstance(); - const rows = await db.all( - 'SELECT * FROM emojis ORDER BY lastUsage DESC LIMIT $limit;', - { - $limit: limit, - } - ); + const rows = db + .prepare( + ` + SELECT * + FROM emojis + ORDER BY lastUsage DESC + LIMIT $limit; + ` + ) + .all({ limit }); return rows || []; } // All data in database -async function removeAll() { +async function removeAll(): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - db.run('DELETE FROM conversations;'), - db.run('DELETE FROM identityKeys;'), - db.run('DELETE FROM items;'), - db.run('DELETE FROM messages;'), - db.run('DELETE FROM preKeys;'), - db.run('DELETE FROM sessions;'), - db.run('DELETE FROM signedPreKeys;'), - db.run('DELETE FROM unprocessed;'), - db.run('DELETE FROM attachment_downloads;'), - db.run('DELETE FROM messages_fts;'), - db.run('DELETE FROM stickers;'), - db.run('DELETE FROM sticker_packs;'), - db.run('DELETE FROM sticker_references;'), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + db.exec(` + DELETE FROM conversations; + DELETE FROM identityKeys; + DELETE FROM items; + DELETE FROM messages; + DELETE FROM preKeys; + DELETE FROM sessions; + DELETE FROM signedPreKeys; + DELETE FROM unprocessed; + DELETE FROM attachment_downloads; + DELETE FROM messages_fts; + DELETE FROM stickers; + DELETE FROM sticker_packs; + DELETE FROM sticker_references; + `); + })(); } -removeAll.needsSerial = true; // Anything that isn't user-visible data -async function removeAllConfiguration() { +async function removeAllConfiguration(): Promise { const db = getInstance(); - await db.run('BEGIN TRANSACTION;'); - try { - await Promise.all([ - db.run('DELETE FROM identityKeys;'), - db.run('DELETE FROM items;'), - db.run('DELETE FROM preKeys;'), - db.run('DELETE FROM sessions;'), - db.run('DELETE FROM signedPreKeys;'), - db.run('DELETE FROM unprocessed;'), - ]); - - await db.run('COMMIT TRANSACTION;'); - } catch (error) { - await db.run('ROLLBACK;'); - throw error; - } + db.transaction(() => { + db.exec(` + DELETE FROM identityKeys; + DELETE FROM items; + DELETE FROM preKeys; + DELETE FROM sessions; + DELETE FROM signedPreKeys; + DELETE FROM unprocessed; + `); + })(); } -removeAllConfiguration.needsSerial = true; async function getMessagesNeedingUpgrade( limit: number, { maxVersion }: { maxVersion: number } -) { +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json FROM messages - WHERE schemaVersion IS NULL OR schemaVersion < $maxVersion - LIMIT $limit;`, - { - $maxVersion: maxVersion, - $limit: limit, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json + FROM messages + WHERE schemaVersion IS NULL OR schemaVersion < $maxVersion + LIMIT $limit; + ` + ) + .all({ + maxVersion, + limit, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } async function getMessagesWithVisualMediaAttachments( conversationId: string, { limit }: { limit: number } -) { +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json FROM messages WHERE - conversationId = $conversationId AND - hasVisualMediaAttachments = 1 - ORDER BY received_at DESC, sent_at DESC - LIMIT $limit;`, - { - $conversationId: conversationId, - $limit: limit, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages WHERE + conversationId = $conversationId AND + hasVisualMediaAttachments = 1 + ORDER BY received_at DESC, sent_at DESC + LIMIT $limit; + ` + ) + .all({ + conversationId, + limit, + }); - return map(rows, row => jsonToObject(row.json)); + return rows.map(row => jsonToObject(row.json)); } async function getMessagesWithFileAttachments( conversationId: string, { limit }: { limit: number } -) { +): Promise> { const db = getInstance(); - const rows = await db.all( - `SELECT json FROM messages WHERE - conversationId = $conversationId AND - hasFileAttachments = 1 - ORDER BY received_at DESC, sent_at DESC - LIMIT $limit;`, - { - $conversationId: conversationId, - $limit: limit, - } - ); + const rows = db + .prepare( + ` + SELECT json FROM messages WHERE + conversationId = $conversationId AND + hasFileAttachments = 1 + ORDER BY received_at DESC, sent_at DESC + LIMIT $limit; + ` + ) + .all({ + conversationId, + limit, + }); return map(rows, row => jsonToObject(row.json)); } -function getExternalFilesForMessage(message: MessageType) { +function getExternalFilesForMessage(message: MessageType): Array { const { attachments, contact, quote, preview, sticker } = message; const files: Array = []; @@ -4269,7 +4105,7 @@ function getExternalFilesForMessage(message: MessageType) { function getExternalFilesForConversation( conversation: Pick -) { +): Array { const { avatar, profileAvatar } = conversation; const files: Array = []; @@ -4286,7 +4122,7 @@ function getExternalFilesForConversation( function getExternalDraftFilesForConversation( conversation: Pick -) { +): Array { const draftAttachments = conversation.draftAttachments || []; const files: Array = []; @@ -4304,14 +4140,16 @@ function getExternalDraftFilesForConversation( return files; } -async function removeKnownAttachments(allAttachments: Array) { +async function removeKnownAttachments( + allAttachments: Array +): Promise> { const db = getInstance(); const lookup: Dictionary = fromPairs( map(allAttachments, file => [file, true]) ); const chunkSize = 50; - const total = await getMessageCount(); + const total = getMessageCount(); console.log( `removeKnownAttachments: About to iterate through ${total} messages` ); @@ -4321,21 +4159,24 @@ async function removeKnownAttachments(allAttachments: Array) { let id: string | number = ''; while (!complete) { - const rows = await db.all( - `SELECT json FROM messages - WHERE id > $id - ORDER BY id ASC - LIMIT $chunkSize;`, - { - $id: id, - $chunkSize: chunkSize, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM messages + WHERE id > $id + ORDER BY id ASC + LIMIT $chunkSize; + ` + ) + .all({ + id, + chunkSize, + }); - const messages: Array = map(rows, row => + const messages: Array = rows.map(row => jsonToObject(row.json) ); - forEach(messages, message => { + messages.forEach(message => { const externalFiles = getExternalFilesForMessage(message); forEach(externalFiles, file => { delete lookup[file]; @@ -4358,29 +4199,32 @@ async function removeKnownAttachments(allAttachments: Array) { // value is still a string but it's smaller than every other string. id = 0; - const conversationTotal = await getConversationCount(); + const conversationTotal = getConversationCount(); console.log( `removeKnownAttachments: About to iterate through ${conversationTotal} conversations` ); while (!complete) { - const rows = await db.all( - `SELECT json FROM conversations - WHERE id > $id - ORDER BY id ASC - LIMIT $chunkSize;`, - { - $id: id, - $chunkSize: chunkSize, - } - ); + const rows = db + .prepare( + ` + SELECT json FROM conversations + WHERE id > $id + ORDER BY id ASC + LIMIT $chunkSize; + ` + ) + .all({ + id, + chunkSize, + }); const conversations: Array = map(rows, row => jsonToObject(row.json) ); - forEach(conversations, conversation => { + conversations.forEach(conversation => { const externalFiles = getExternalFilesForConversation(conversation); - forEach(externalFiles, file => { + externalFiles.forEach(file => { delete lookup[file]; }); }); @@ -4398,14 +4242,16 @@ async function removeKnownAttachments(allAttachments: Array) { return Object.keys(lookup); } -async function removeKnownStickers(allStickers: Array) { +async function removeKnownStickers( + allStickers: Array +): Promise> { const db = getInstance(); const lookup: Dictionary = fromPairs( map(allStickers, file => [file, true]) ); const chunkSize = 50; - const total = await getStickerCount(); + const total = getStickerCount(); console.log( `removeKnownStickers: About to iterate through ${total} stickers` ); @@ -4415,23 +4261,26 @@ async function removeKnownStickers(allStickers: Array) { let rowid = 0; while (!complete) { - const rows = await db.all( - `SELECT rowid, path FROM stickers - WHERE rowid > $rowid - ORDER BY rowid ASC - LIMIT $chunkSize;`, - { - $rowid: rowid, - $chunkSize: chunkSize, - } - ); + const rows: Array<{ rowid: number; path: string }> = db + .prepare( + ` + SELECT rowid, path FROM stickers + WHERE rowid > $rowid + ORDER BY rowid ASC + LIMIT $chunkSize; + ` + ) + .all({ + rowid, + chunkSize, + }); - const files: Array = map(rows, row => row.path); - forEach(files, file => { + const files: Array = rows.map(row => row.path); + files.forEach(file => { delete lookup[file]; }); - const lastSticker: StickerType | undefined = last(rows); + const lastSticker = last(rows); if (lastSticker) { ({ rowid } = lastSticker); } @@ -4444,14 +4293,16 @@ async function removeKnownStickers(allStickers: Array) { return Object.keys(lookup); } -async function removeKnownDraftAttachments(allStickers: Array) { +async function removeKnownDraftAttachments( + allStickers: Array +): Promise> { const db = getInstance(); const lookup: Dictionary = fromPairs( map(allStickers, file => [file, true]) ); const chunkSize = 50; - const total = await getConversationCount(); + const total = getConversationCount(); console.log( `removeKnownDraftAttachments: About to iterate through ${total} conversations` ); @@ -4460,26 +4311,29 @@ async function removeKnownDraftAttachments(allStickers: Array) { let count = 0; // Though conversations.id is a string, this ensures that, when coerced, this // value is still a string but it's smaller than every other string. - let id = 0; + let id: number | string = 0; while (!complete) { - const rows = await db.all( - `SELECT json FROM conversations - WHERE id > $id - ORDER BY id ASC - LIMIT $chunkSize;`, - { - $id: id, - $chunkSize: chunkSize, - } - ); + const rows: JSONRows = db + .prepare( + ` + SELECT json FROM conversations + WHERE id > $id + ORDER BY id ASC + LIMIT $chunkSize; + ` + ) + .all({ + id, + chunkSize, + }); - const conversations: Array = map(rows, row => + const conversations: Array = rows.map(row => jsonToObject(row.json) ); - forEach(conversations, conversation => { + conversations.forEach(conversation => { const externalFiles = getExternalDraftFilesForConversation(conversation); - forEach(externalFiles, file => { + externalFiles.forEach(file => { delete lookup[file]; }); }); diff --git a/ts/sql/main.ts b/ts/sql/main.ts new file mode 100644 index 0000000000..75ac3b7589 --- /dev/null +++ b/ts/sql/main.ts @@ -0,0 +1,113 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { join } from 'path'; +import { Worker } from 'worker_threads'; + +export type InitializeOptions = { + readonly configDir: string; + readonly key: string; +}; + +export type WorkerRequest = + | { + readonly type: 'init'; + readonly options: InitializeOptions; + } + | { + readonly type: 'close'; + } + | { + readonly type: 'sqlCall'; + readonly method: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readonly args: ReadonlyArray; + }; + +export type WrappedWorkerRequest = { + readonly seq: number; + readonly request: WorkerRequest; +}; + +export type WrappedWorkerResponse = { + readonly seq: number; + readonly error: string | undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readonly response: any; +}; + +type PromisePair = { + resolve: (response: T) => void; + reject: (error: Error) => void; +}; + +export class MainSQL { + private readonly worker: Worker; + + private readonly onExit: Promise; + + private seq = 0; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private onResponse = new Map>(); + + constructor() { + const appDir = join(__dirname, '..', '..').replace( + /app\.asar$/, + 'app.asar.unpacked' + ); + + this.worker = new Worker(join(appDir, 'ts', 'sql', 'mainWorker.js')); + + this.worker.on('message', (wrappedResponse: WrappedWorkerResponse) => { + const { seq, error, response } = wrappedResponse; + + const pair = this.onResponse.get(seq); + this.onResponse.delete(seq); + if (!pair) { + throw new Error(`Unexpected worker response with seq: ${seq}`); + } + + if (error) { + pair.reject(new Error(error)); + } else { + pair.resolve(response); + } + }); + + this.onExit = new Promise(resolve => { + this.worker.once('exit', resolve); + }); + } + + public async initialize(options: InitializeOptions): Promise { + return this.send({ type: 'init', options }); + } + + public async close(): Promise { + await this.send({ type: 'close' }); + await this.onExit; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public async sqlCall(method: string, args: ReadonlyArray): Promise { + return this.send({ type: 'sqlCall', method, args }); + } + + private async send(request: WorkerRequest): Promise { + const { seq } = this; + this.seq += 1; + + const result = new Promise((resolve, reject) => { + this.onResponse.set(seq, { resolve, reject }); + }); + + const wrappedRequest: WrappedWorkerRequest = { + seq, + request, + }; + this.worker.postMessage(wrappedRequest); + + return result; + } +} diff --git a/ts/sql/mainWorker.ts b/ts/sql/mainWorker.ts new file mode 100644 index 0000000000..466be06e18 --- /dev/null +++ b/ts/sql/mainWorker.ts @@ -0,0 +1,56 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { parentPort } from 'worker_threads'; + +import { WrappedWorkerRequest, WrappedWorkerResponse } from './main'; +import db from './Server'; + +if (!parentPort) { + throw new Error('Must run as a worker thread'); +} + +const port = parentPort; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function respond(seq: number, error: Error | undefined, response?: any) { + const wrappedResponse: WrappedWorkerResponse = { + seq, + error: error ? error.stack : undefined, + response, + }; + port.postMessage(wrappedResponse); +} + +port.on('message', async ({ seq, request }: WrappedWorkerRequest) => { + try { + if (request.type === 'init') { + await db.initialize(request.options); + + respond(seq, undefined, undefined); + return; + } + + if (request.type === 'close') { + await db.close(); + + respond(seq, undefined, undefined); + process.exit(0); + return; + } + + if (request.type === 'sqlCall') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const method = (db as any)[request.method]; + if (typeof method !== 'function') { + throw new Error(`Invalid sql method: ${method}`); + } + + respond(seq, undefined, await method.apply(db, request.args)); + } else { + throw new Error('Unexpected request type'); + } + } catch (error) { + respond(seq, error, undefined); + } +}); diff --git a/ts/sqlcipher.d.ts b/ts/sqlcipher.d.ts deleted file mode 100644 index 0ddcb43b04..0000000000 --- a/ts/sqlcipher.d.ts +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2020-2021 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only - -// Taken from: -// https://github.com/DefinitelyTyped/DefinitelyTyped/blob/8bf8aedba75ada257428c4846d2bc7d14e3b4be8/types/sqlite3/index.d.ts - -declare module '@journeyapps/sqlcipher' { - // Type definitions for sqlite3 3.1 - // Project: http://github.com/mapbox/node-sqlite3 - // Definitions by: Nick Malaguti - // Sumant Manne - // Behind The Math - // Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped - - /// - - import events = require('events'); - - export const OPEN_READONLY: number; - export const OPEN_READWRITE: number; - export const OPEN_CREATE: number; - export const OPEN_SHAREDCACHE: number; - export const OPEN_PRIVATECACHE: number; - export const OPEN_URI: number; - - export const cached: { - Database( - filename: string, - callback?: (this: Database, err: Error | null) => void - ): Database; - Database( - filename: string, - mode?: number, - callback?: (this: Database, err: Error | null) => void - ): Database; - }; - - export type RunResult = Statement & { - lastID: number; - changes: number; - }; - - export class Statement { - bind(callback?: (err: Error | null) => void): this; - bind(...params: any[]): this; - - reset(callback?: (err: null) => void): this; - - finalize(callback?: (err: Error) => void): Database; - - run(callback?: (err: Error | null) => void): this; - run( - params: any, - callback?: (this: RunResult, err: Error | null) => void - ): this; - run(...params: any[]): this; - - get(callback?: (err: Error | null, row?: any) => void): this; - get( - params: any, - callback?: (this: RunResult, err: Error | null, row?: any) => void - ): this; - get(...params: any[]): this; - - all(callback?: (err: Error | null, rows: any[]) => void): this; - all( - params: any, - callback?: (this: RunResult, err: Error | null, rows: any[]) => void - ): this; - all(...params: any[]): this; - - each( - callback?: (err: Error | null, row: any) => void, - complete?: (err: Error | null, count: number) => void - ): this; - each( - params: any, - callback?: (this: RunResult, err: Error | null, row: any) => void, - complete?: (err: Error | null, count: number) => void - ): this; - each(...params: any[]): this; - } - - export class Database extends events.EventEmitter { - constructor(filename: string, callback?: (err: Error | null) => void); - constructor( - filename: string, - mode?: number, - callback?: (err: Error | null) => void - ); - - close(callback?: (err: Error | null) => void): void; - - run( - sql: string, - callback?: (this: RunResult, err: Error | null) => void - ): this; - run( - sql: string, - params: any, - callback?: (this: RunResult, err: Error | null) => void - ): this; - run(sql: string, ...params: any[]): this; - - get( - sql: string, - callback?: (this: Statement, err: Error | null, row: any) => void - ): this; - get( - sql: string, - params: any, - callback?: (this: Statement, err: Error | null, row: any) => void - ): this; - get(sql: string, ...params: any[]): this; - - all( - sql: string, - callback?: (this: Statement, err: Error | null, rows: any[]) => void - ): this; - all( - sql: string, - params: any, - callback?: (this: Statement, err: Error | null, rows: any[]) => void - ): this; - all(sql: string, ...params: any[]): this; - - each( - sql: string, - callback?: (this: Statement, err: Error | null, row: any) => void, - complete?: (err: Error | null, count: number) => void - ): this; - each( - sql: string, - params: any, - callback?: (this: Statement, err: Error | null, row: any) => void, - complete?: (err: Error | null, count: number) => void - ): this; - each(sql: string, ...params: any[]): this; - - exec( - sql: string, - callback?: (this: Statement, err: Error | null) => void - ): this; - - prepare( - sql: string, - callback?: (this: Statement, err: Error | null) => void - ): Statement; - prepare( - sql: string, - params: any, - callback?: (this: Statement, err: Error | null) => void - ): Statement; - prepare(sql: string, ...params: any[]): Statement; - - serialize(callback?: () => void): void; - parallelize(callback?: () => void): void; - - on(event: 'trace', listener: (sql: string) => void): this; - on(event: 'profile', listener: (sql: string, time: number) => void): this; - on(event: 'error', listener: (err: Error) => void): this; - on(event: 'open' | 'close', listener: () => void): this; - on(event: string, listener: (...args: any[]) => void): this; - - configure(option: 'busyTimeout', value: number): void; - interrupt(): void; - } - - export function verbose(): sqlite3; - - export interface sqlite3 { - OPEN_READONLY: number; - OPEN_READWRITE: number; - OPEN_CREATE: number; - OPEN_SHAREDCACHE: number; - OPEN_PRIVATECACHE: number; - OPEN_URI: number; - cached: typeof cached; - RunResult: RunResult; - Statement: typeof Statement; - Database: typeof Database; - verbose(): this; - } -} diff --git a/ts/state/ducks/conversations.ts b/ts/state/ducks/conversations.ts index 6e356c56cc..067e0171f9 100644 --- a/ts/state/ducks/conversations.ts +++ b/ts/state/ducks/conversations.ts @@ -42,7 +42,7 @@ import { toggleSelectedContactForGroupAddition } from '../../groups/toggleSelect export type DBConversationType = { id: string; activeAt?: number; - lastMessage: string; + lastMessage?: string | null; type: string; }; @@ -138,19 +138,28 @@ export type ConversationType = { export type ConversationLookupType = { [key: string]: ConversationType; }; +export type CustomError = Error & { + identifier?: string; + number?: string; +}; export type MessageType = { id: string; conversationId: string; source?: string; sourceUuid?: string; - type: + type?: | 'incoming' | 'outgoing' | 'group' | 'keychange' | 'verified-change' | 'message-history-unsynced' - | 'call-history'; + | 'call-history' + | 'chat-session-refreshed' + | 'group-v1-migration' + | 'group-v2-change' + | 'profile-change' + | 'timer-notification'; quote?: { author?: string; authorUuid?: string }; received_at: number; sent_at?: number; @@ -179,7 +188,7 @@ export type MessageType = { }>; deletedForEveryone?: boolean; - errors?: Array; + errors?: Array; group_update?: unknown; callHistoryDetails?: CallHistoryDetailsFromDiskType; diff --git a/ts/state/ducks/search.ts b/ts/state/ducks/search.ts index 3a27d9c283..de327c7371 100644 --- a/ts/state/ducks/search.ts +++ b/ts/state/ducks/search.ts @@ -5,6 +5,10 @@ import { omit, reject } from 'lodash'; import { normalize } from '../../types/PhoneNumber'; import { cleanSearchTerm } from '../../util/cleanSearchTerm'; +import { + ClientSearchResultMessageType, + ClientInterface, +} from '../../sql/Interface'; import dataInterface from '../../sql/Client'; import { makeLookup } from '../../util/makeLookup'; import { BodyRangesType } from '../../types/Util'; @@ -23,7 +27,7 @@ const { searchConversations: dataSearchConversations, searchMessages: dataSearchMessages, searchMessagesInConversation, -} = dataInterface; +}: ClientInterface = dataInterface; // State @@ -244,7 +248,10 @@ function updateSearchTerm(query: string): UpdateSearchTermActionType { }; } -async function queryMessages(query: string, searchConversationId?: string) { +async function queryMessages( + query: string, + searchConversationId?: string +): Promise> { try { const normalized = cleanSearchTerm(query); diff --git a/ts/util/combineNames.ts b/ts/util/combineNames.ts index c892c3927e..86d7134bd3 100644 --- a/ts/util/combineNames.ts +++ b/ts/util/combineNames.ts @@ -36,7 +36,7 @@ const Hangul_Syllables = /[\uAC00-\uD7AF]/; const isIdeographic = /[\u3006\u3007\u3021-\u3029\u3038-\u303A\u3400-\u4DB5\u4E00-\u9FEF\uF900-\uFA6D\uFA70-\uFAD9]|[\uD81C-\uD820\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872\uD874-\uD879][\uDC00-\uDFFF]|\uD821[\uDC00-\uDFF7]|\uD822[\uDC00-\uDEF2]|\uD82C[\uDD70-\uDEFB]|\uD869[\uDC00-\uDED6\uDF00-\uDFFF]|\uD86D[\uDC00-\uDF34\uDF40-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEA1\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0]|\uD87E[\uDC00-\uDE1D]/; export function combineNames( - given: string, + given?: string, family?: string ): undefined | string { if (!given) { diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 779fb483be..1d7c7f1720 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -10236,78 +10236,6 @@ "reasonCategory": "falseMatch", "updated": "2019-07-19T17:16:02.404Z" }, - { - "rule": "jQuery-load(", - "path": "node_modules/node-pre-gyp/node_modules/debug/dist/debug.js", - "line": " createDebug.enable(createDebug.load());", - "lineNumber": 694, - "reasonCategory": "falseMatch", - "updated": "2021-01-21T16:16:34.352Z" - }, - { - "rule": "jQuery-load(", - "path": "node_modules/node-pre-gyp/node_modules/debug/dist/debug.js", - "line": " function load() {", - "lineNumber": 828, - "reasonCategory": "falseMatch", - "updated": "2021-01-21T16:16:34.352Z" - }, - { - "rule": "jQuery-load(", - "path": "node_modules/node-pre-gyp/node_modules/debug/src/browser.js", - "line": "function load() {", - "lineNumber": 129, - "reasonCategory": "falseMatch", - "updated": "2021-01-21T16:16:34.352Z" - }, - { - "rule": "jQuery-load(", - "path": "node_modules/node-pre-gyp/node_modules/debug/src/common.js", - "line": " createDebug.enable(createDebug.load());", - "lineNumber": 244, - "reasonCategory": "falseMatch", - "updated": "2021-01-21T16:16:34.352Z" - }, - { - "rule": "jQuery-load(", - "path": "node_modules/node-pre-gyp/node_modules/debug/src/node.js", - "line": "function load() {", - "lineNumber": 135, - "reasonCategory": "falseMatch", - "updated": "2021-01-21T16:16:34.352Z" - }, - { - "rule": "jQuery-append(", - "path": "node_modules/node-pre-gyp/node_modules/needle/lib/multipart.js", - "line": " function append(data, filename) {", - "lineNumber": 42, - "reasonCategory": "falseMatch", - "updated": "2020-04-30T22:35:27.860Z" - }, - { - "rule": "jQuery-append(", - "path": "node_modules/node-pre-gyp/node_modules/needle/lib/multipart.js", - "line": " if (part.buffer) return append(part.buffer, filename);", - "lineNumber": 58, - "reasonCategory": "falseMatch", - "updated": "2020-04-30T22:35:27.860Z" - }, - { - "rule": "jQuery-append(", - "path": "node_modules/node-pre-gyp/node_modules/needle/lib/multipart.js", - "line": " append(data, filename);", - "lineNumber": 62, - "reasonCategory": "falseMatch", - "updated": "2020-04-30T22:35:27.860Z" - }, - { - "rule": "jQuery-append(", - "path": "node_modules/node-pre-gyp/node_modules/needle/lib/multipart.js", - "line": " append();", - "lineNumber": 77, - "reasonCategory": "falseMatch", - "updated": "2020-04-30T22:35:27.860Z" - }, { "rule": "jQuery-$(", "path": "node_modules/nugget/node_modules/ajv/dist/ajv.min.js", @@ -15681,7 +15609,7 @@ "rule": "jQuery-load(", "path": "ts/LibSignalStore.ts", "line": " await window.ConversationController.load();", - "lineNumber": 1222, + "lineNumber": 1190, "reasonCategory": "falseMatch", "updated": "2021-02-27T00:48:49.313Z" }, @@ -16785,4 +16713,4 @@ "updated": "2021-01-08T15:46:32.143Z", "reasonDetail": "Doesn't manipulate the DOM. This is just a function." } -] \ No newline at end of file +] diff --git a/ts/util/lint/linter.ts b/ts/util/lint/linter.ts index 217b0c89ac..6d98fd6865 100644 --- a/ts/util/lint/linter.ts +++ b/ts/util/lint/linter.ts @@ -44,6 +44,8 @@ const basePath = join(__dirname, '../../..'); const searchPattern = normalizePath(join(basePath, '**/*.{js,ts,tsx}')); const excludedFilesRegexps = [ + '^release/', + // Non-distributed files '\\.d\\.ts$', diff --git a/ts/views/conversation_view.ts b/ts/views/conversation_view.ts index 524caaa3f4..5dbefe487c 100644 --- a/ts/views/conversation_view.ts +++ b/ts/views/conversation_view.ts @@ -1138,7 +1138,7 @@ Whisper.ConversationView = Whisper.View.extend({ // If newest in-memory message is unread, scrolling down would mean going to // the very bottom, not the oldest unread. - if (newestInMemoryMessage.isUnread()) { + if (newestInMemoryMessage && newestInMemoryMessage.isUnread()) { scrollToLatestUnread = false; } } @@ -3247,9 +3247,13 @@ Whisper.ConversationView = Whisper.View.extend({ ? await getMessageById(messageId, { Message: Whisper.Message, }) - : null; + : undefined; try { + if (!messageModel) { + throw new Error('Message not found'); + } + await this.model.sendReactionMessage(reaction, { targetAuthorUuid: messageModel.getSourceUuid(), targetTimestamp: messageModel.get('sent_at'), @@ -3329,7 +3333,7 @@ Whisper.ConversationView = Whisper.View.extend({ ? await getMessageById(messageId, { Message: Whisper.Message, }) - : null; + : undefined; if (model && !model.canReply()) { return; diff --git a/yarn.lock b/yarn.lock index 8c88e6cdcd..670cae1eac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1348,13 +1348,6 @@ resolved "https://registry.yarnpkg.com/@icons/material/-/material-0.2.4.tgz#e90c9f71768b3736e76d7dd6783fc6c2afa88bc8" integrity sha512-QPcGmICAPbGLGb6F/yNf/KzKqvFx8z5qx3D1yFqVAjoFmXK35EgyW+cJ57Te3CNsmzblwtzakLGFqHPqrfb4Tw== -"@journeyapps/sqlcipher@https://github.com/EvanHahn-signal/node-sqlcipher.git#16916949f0c010f6e6d3d5869b10a0ab813eae75": - version "5.0.0" - resolved "https://github.com/EvanHahn-signal/node-sqlcipher.git#16916949f0c010f6e6d3d5869b10a0ab813eae75" - dependencies: - node-addon-api "^3.0.0" - node-pre-gyp "^0.15.0" - "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde" @@ -2143,6 +2136,13 @@ "@types/jquery" "*" "@types/underscore" "*" +"@types/better-sqlite3@5.4.1": + version "5.4.1" + resolved "https://registry.yarnpkg.com/@types/better-sqlite3/-/better-sqlite3-5.4.1.tgz#d45600bc19f8f41397263d037ca9b0d05df85e58" + integrity sha512-8hje3Rhsg/9veTkALfCwiWn7VMrP1QDwHhBSgerttYPABEvrHsMQnU9dlqoM6QX3x4uw3Y06dDVz8uDQo1J4Ng== + dependencies: + "@types/integer" "*" + "@types/blueimp-load-image@5.14.1": version "5.14.1" resolved "https://registry.yarnpkg.com/@types/blueimp-load-image/-/blueimp-load-image-5.14.1.tgz#3963813699b574e757a140ed75a51050177ac780" @@ -2323,6 +2323,11 @@ dependencies: "@types/node" "*" +"@types/integer@*": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/integer/-/integer-1.0.1.tgz#025d87e30d97f539fcc6087372af7d3672ffbbe6" + integrity sha512-DmZDpSVnsuBrOhtHwE1oKmUJ3qVjHhhNQ7WnZy9/RhH3A24Ar+9o4SoaCWcTzQhalpRDIAMsfdoZLWNJtdBR7A== + "@types/jquery@*", "@types/jquery@3.5.0": version "3.5.0" resolved "https://registry.yarnpkg.com/@types/jquery/-/jquery-3.5.0.tgz#ccb7dfd317d02d4227dd3803c75297d0c10dad68" @@ -4198,6 +4203,13 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +"better-sqlite3@https://github.com/indutny/better-sqlite3#a78376d86b5856c14ab4e2f3995f41e1f80df846": + version "7.1.4" + resolved "https://github.com/indutny/better-sqlite3#a78376d86b5856c14ab4e2f3995f41e1f80df846" + dependencies: + bindings "^1.5.0" + tar "^6.1.0" + big.js@^3.1.3: version "3.2.0" resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e" @@ -4911,6 +4923,11 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + chrome-trace-event@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz#234090ee97c7d4ad1a2c4beae27505deffc608a4" @@ -5874,7 +5891,7 @@ debug@0.7.4: version "0.7.4" resolved "https://registry.yarnpkg.com/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39" -debug@2, debug@2.6.9, debug@^2.1.2, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9: +debug@2, debug@2.6.9, debug@^2.3.3, debug@^2.6.0, debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -7984,6 +8001,13 @@ fs-minipass@^1.2.5: dependencies: minipass "^2.2.1" +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + fs-write-stream-atomic@^1.0.8: version "1.0.10" resolved "https://registry.yarnpkg.com/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz#b47df53493ef911df75731e70a9ded0189db40c9" @@ -11157,13 +11181,12 @@ minipass@^2.3.4, minipass@^2.3.5: safe-buffer "^5.1.2" yallist "^3.0.0" -minipass@^2.8.6: - version "2.9.0" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6" - integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg== +minipass@^3.0.0: + version "3.1.3" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd" + integrity sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg== dependencies: - safe-buffer "^5.1.2" - yallist "^3.0.0" + yallist "^4.0.0" minizlib@^1.1.0: version "1.1.0" @@ -11185,6 +11208,14 @@ minizlib@^1.2.1: dependencies: minipass "^2.2.1" +minizlib@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" + integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mississippi@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022" @@ -11241,13 +11272,18 @@ mkdirp@0.5.2: dependencies: minimist "^1.2.5" -mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@~0.5.1: +mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== dependencies: minimist "^1.2.5" +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + mkpath@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/mkpath/-/mkpath-0.1.0.tgz#7554a6f8d871834cc97b5462b122c4c124d6de91" @@ -11434,15 +11470,7 @@ nconf@^0.10.0: secure-keys "^1.0.0" yargs "^3.19.0" -needle@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/needle/-/needle-2.2.1.tgz#b5e325bd3aae8c2678902fa296f729455d1d3a7d" - dependencies: - debug "^2.1.2" - iconv-lite "^0.4.4" - sax "^1.2.4" - -needle@^2.2.4, needle@^2.3.3, needle@^2.4.0: +needle@^2.2.1, needle@^2.2.4, needle@^2.3.3, needle@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/needle/-/needle-2.4.1.tgz#14af48732463d7475696f937626b1b993247a56a" integrity sha512-x/gi6ijr4B7fwl6WYL9FwlCvRQKGlUNvnceho8wxkwXqN8jvVmmmATTmZPRRG7b/yC1eode26C2HO9jl78Du9g== @@ -11451,15 +11479,6 @@ needle@^2.2.4, needle@^2.3.3, needle@^2.4.0: iconv-lite "^0.4.4" sax "^1.2.4" -needle@^2.5.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/needle/-/needle-2.6.0.tgz#24dbb55f2509e2324b4a99d61f413982013ccdbe" - integrity sha512-KKYdza4heMsEfSWD7VPUIz3zX2XDwOyX2d+geb4vrERZMT5RMU6ujjaD+I5Yr54uZxQ2w6XRTAhHBbSCyovZBg== - dependencies: - debug "^3.2.6" - iconv-lite "^0.4.4" - sax "^1.2.4" - negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" @@ -11645,22 +11664,6 @@ node-pre-gyp@^0.12.0: semver "^5.3.0" tar "^4" -node-pre-gyp@^0.15.0: - version "0.15.0" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.15.0.tgz#c2fc383276b74c7ffa842925241553e8b40f1087" - integrity sha512-7QcZa8/fpaU/BKenjcaeFF9hLz2+7S9AqyXFhlH/rilsQ/hPZKK32RtR5EQHJElgu+q5RfbJ34KriI79UWaorA== - dependencies: - detect-libc "^1.0.2" - mkdirp "^0.5.3" - needle "^2.5.0" - nopt "^4.0.1" - npm-packlist "^1.1.6" - npmlog "^4.0.2" - rc "^1.2.7" - rimraf "^2.6.1" - semver "^5.3.0" - tar "^4.4.2" - node-releases@^1.1.25: version "1.1.27" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.27.tgz#b19ec8add2afe9a826a99dceccc516104c1edaf4" @@ -16166,19 +16169,6 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" -tar@^4.4.2: - version "4.4.13" - resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" - integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA== - dependencies: - chownr "^1.1.1" - fs-minipass "^1.2.5" - minipass "^2.8.6" - minizlib "^1.2.1" - mkdirp "^0.5.0" - safe-buffer "^5.1.2" - yallist "^3.0.3" - tar@^4.4.8: version "4.4.10" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.10.tgz#946b2810b9a5e0b26140cf78bea6b0b0d689eba1" @@ -16192,6 +16182,18 @@ tar@^4.4.8: safe-buffer "^5.1.2" yallist "^3.0.3" +tar@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83" + integrity sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + telejson@^2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/telejson/-/telejson-2.2.2.tgz#d61d721d21849a6e4070d547aab302a9bd22c720"