diff --git a/app/SystemTraySettingCache.ts b/app/SystemTraySettingCache.ts index 19a3b843ea..aff9e1ae33 100644 --- a/app/SystemTraySettingCache.ts +++ b/app/SystemTraySettingCache.ts @@ -62,7 +62,7 @@ export class SystemTraySettingCache { const value = fastValue ?? - (await this.sql.sqlCall('getItemById', ['system-tray-setting']))?.value; + (await this.sql.sqlCall('getItemById', 'system-tray-setting'))?.value; if (value !== undefined) { result = parseSystemTraySetting(value); diff --git a/app/attachment_channel.ts b/app/attachment_channel.ts index 35efb1de55..40fcd9803f 100644 --- a/app/attachment_channel.ts +++ b/app/attachment_channel.ts @@ -4,11 +4,22 @@ import { ipcMain } from 'electron'; import * as rimraf from 'rimraf'; import { + getAllAttachments, getPath, getStickersPath, getTempPath, getDraftPath, + deleteAll as deleteAllAttachments, + deleteAllBadges, + getAllStickers, + deleteAllStickers, + getAllDraftAttachments, + deleteAllDraftAttachments, } from './attachments'; +import type { MainSQL } from '../ts/sql/main'; +import type { MessageAttachmentsCursorType } from '../ts/sql/Interface'; +import * as Errors from '../ts/types/errors'; +import { sleep } from '../ts/util/sleep'; let initialized = false; @@ -18,12 +29,140 @@ const ERASE_TEMP_KEY = 'erase-temp'; const ERASE_DRAFTS_KEY = 'erase-drafts'; const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments'; +const INTERACTIVITY_DELAY = 50; + +type DeleteOrphanedAttachmentsOptionsType = Readonly<{ + orphanedAttachments: Set; + sql: MainSQL; + userDataPath: string; +}>; + +type CleanupOrphanedAttachmentsOptionsType = Readonly<{ + sql: MainSQL; + userDataPath: string; +}>; + +async function cleanupOrphanedAttachments({ + sql, + userDataPath, +}: CleanupOrphanedAttachmentsOptionsType): Promise { + await deleteAllBadges({ + userDataPath, + pathsToKeep: await sql.sqlCall('getAllBadgeImageFileLocalPaths'), + }); + + const allStickers = await getAllStickers(userDataPath); + const orphanedStickers = await sql.sqlCall( + 'removeKnownStickers', + allStickers + ); + await deleteAllStickers({ + userDataPath, + stickers: orphanedStickers, + }); + + const allDraftAttachments = await getAllDraftAttachments(userDataPath); + const orphanedDraftAttachments = await sql.sqlCall( + 'removeKnownDraftAttachments', + allDraftAttachments + ); + await deleteAllDraftAttachments({ + userDataPath, + attachments: orphanedDraftAttachments, + }); + + // Delete orphaned attachments from conversations and messages. + + const orphanedAttachments = new Set(await getAllAttachments(userDataPath)); + + { + const attachments: ReadonlyArray = await sql.sqlCall( + 'getKnownConversationAttachments' + ); + + for (const known of attachments) { + orphanedAttachments.delete(known); + } + } + + // This call is intentionally not awaited. We block the app while running + // all fetches above to ensure that there are no in-flight attachments that + // are saved to disk, but not put into any message or conversation model yet. + deleteOrphanedAttachments({ + orphanedAttachments, + sql, + userDataPath, + }); +} + +function deleteOrphanedAttachments({ + orphanedAttachments, + sql, + userDataPath, +}: DeleteOrphanedAttachmentsOptionsType): void { + // This function *can* throw. + async function runWithPossibleException(): Promise { + let cursor: MessageAttachmentsCursorType | undefined; + try { + do { + let attachments: ReadonlyArray; + + // eslint-disable-next-line no-await-in-loop + ({ attachments, cursor } = await sql.sqlCall( + 'getKnownMessageAttachments', + cursor + )); + + for (const known of attachments) { + orphanedAttachments.delete(known); + } + + if (cursor === undefined) { + break; + } + + // Let other SQL calls come through. There are hundreds of thousands of + // messages in the database and it might take time to go through them all. + // eslint-disable-next-line no-await-in-loop + await sleep(INTERACTIVITY_DELAY); + } while (cursor !== undefined && !cursor.done); + } finally { + if (cursor !== undefined) { + await sql.sqlCall('finishGetKnownMessageAttachments', cursor); + } + } + + await deleteAllAttachments({ + userDataPath, + attachments: Array.from(orphanedAttachments), + }); + } + + async function runSafe() { + const start = Date.now(); + try { + await runWithPossibleException(); + } catch (error) { + console.error( + 'deleteOrphanedAttachments: error', + Errors.toLogFormat(error) + ); + } finally { + const duration = Date.now() - start; + console.log(`deleteOrphanedAttachments: took ${duration}ms`); + } + } + + // Intentionally not awaiting + runSafe(); +} + export function initialize({ configDir, - cleanupOrphanedAttachments, + sql, }: { configDir: string; - cleanupOrphanedAttachments: () => Promise; + sql: MainSQL; }): void { if (initialized) { throw new Error('initialze: Already initialized!'); @@ -35,58 +174,15 @@ export function initialize({ const tempDir = getTempPath(configDir); const draftDir = getDraftPath(configDir); - ipcMain.on(ERASE_TEMP_KEY, event => { - try { - rimraf.sync(tempDir); - event.sender.send(`${ERASE_TEMP_KEY}-done`); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log(`erase temp error: ${errorForDisplay}`); - event.sender.send(`${ERASE_TEMP_KEY}-done`, error); - } - }); + ipcMain.handle(ERASE_TEMP_KEY, () => rimraf.sync(tempDir)); + ipcMain.handle(ERASE_ATTACHMENTS_KEY, () => rimraf.sync(attachmentsDir)); + ipcMain.handle(ERASE_STICKERS_KEY, () => rimraf.sync(stickersDir)); + ipcMain.handle(ERASE_DRAFTS_KEY, () => rimraf.sync(draftDir)); - ipcMain.on(ERASE_ATTACHMENTS_KEY, event => { - try { - rimraf.sync(attachmentsDir); - event.sender.send(`${ERASE_ATTACHMENTS_KEY}-done`); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log(`erase attachments error: ${errorForDisplay}`); - event.sender.send(`${ERASE_ATTACHMENTS_KEY}-done`, error); - } - }); - - ipcMain.on(ERASE_STICKERS_KEY, event => { - try { - rimraf.sync(stickersDir); - event.sender.send(`${ERASE_STICKERS_KEY}-done`); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log(`erase stickers error: ${errorForDisplay}`); - event.sender.send(`${ERASE_STICKERS_KEY}-done`, error); - } - }); - - ipcMain.on(ERASE_DRAFTS_KEY, event => { - try { - rimraf.sync(draftDir); - event.sender.send(`${ERASE_DRAFTS_KEY}-done`); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log(`erase drafts error: ${errorForDisplay}`); - event.sender.send(`${ERASE_DRAFTS_KEY}-done`, error); - } - }); - - ipcMain.on(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async event => { - try { - await cleanupOrphanedAttachments(); - event.sender.send(`${CLEANUP_ORPHANED_ATTACHMENTS_KEY}-done`); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log(`cleanup orphaned attachments error: ${errorForDisplay}`); - event.sender.send(`${CLEANUP_ORPHANED_ATTACHMENTS_KEY}-done`, error); - } + ipcMain.handle(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async () => { + const start = Date.now(); + await cleanupOrphanedAttachments({ sql, userDataPath: configDir }); + const duration = Date.now() - start; + console.log(`cleanupOrphanedAttachments: took ${duration}ms`); }); } diff --git a/app/main.ts b/app/main.ts index cb38433d27..3f7a2e360f 100644 --- a/app/main.ts +++ b/app/main.ts @@ -224,17 +224,17 @@ let sqlInitTimeEnd = 0; const sql = new MainSQL(); const heicConverter = getHeicConverter(); -async function getSpellCheckSetting() { +async function getSpellCheckSetting(): Promise { const fastValue = ephemeralConfig.get('spell-check'); - if (fastValue !== undefined) { + if (typeof fastValue === 'boolean') { getLogger().info('got fast spellcheck setting', fastValue); return fastValue; } - const json = await sql.sqlCall('getItemById', ['spell-check']); + const json = await sql.sqlCall('getItemById', 'spell-check'); // Default to `true` if setting doesn't exist yet - const slowValue = json ? json.value : true; + const slowValue = typeof json?.value === 'boolean' ? json.value : true; ephemeralConfig.set('spell-check', slowValue); @@ -260,7 +260,7 @@ async function getThemeSetting({ return 'system'; } - const json = await sql.sqlCall('getItemById', ['theme-setting']); + const json = await sql.sqlCall('getItemById', 'theme-setting'); // Default to `system` if setting doesn't exist or is invalid const setting: unknown = json?.value; @@ -928,7 +928,7 @@ async function createWindow() { } // Renderer asks if we are done with the database -ipc.on('database-ready', async event => { +ipc.handle('database-ready', async () => { if (!sqlInitPromise) { getLogger().error('database-ready requested, but sqlInitPromise is falsey'); return; @@ -944,7 +944,6 @@ ipc.on('database-ready', async event => { } getLogger().info('sending `database-ready`'); - event.sender.send('database-ready'); }); ipc.on('show-window', () => { @@ -1259,8 +1258,8 @@ async function showSettingsWindow() { async function getIsLinked() { try { - const number = await sql.sqlCall('getItemById', ['number_id']); - const password = await sql.sqlCall('getItemById', ['password']); + const number = await sql.sqlCall('getItemById', 'number_id'); + const password = await sql.sqlCall('getItemById', 'password'); return Boolean(number && password); } catch (e) { return false; @@ -1651,12 +1650,10 @@ app.on('ready', async () => { // Update both stores ephemeralConfig.set('system-tray-setting', newValue); - await sql.sqlCall('createOrUpdateItem', [ - { - id: 'system-tray-setting', - value: newValue, - }, - ]); + await sql.sqlCall('createOrUpdateItem', { + id: 'system-tray-setting', + value: newValue, + }); if (OS.isWindows()) { getLogger().info('app.ready: enabling open at login'); @@ -1806,8 +1803,8 @@ app.on('ready', async () => { // Initialize IPC channels before creating the window attachmentChannel.initialize({ + sql, configDir: userDataPath, - cleanupOrphanedAttachments, }); sqlChannels.initialize(sql); PowerChannel.initialize({ @@ -1835,10 +1832,10 @@ app.on('ready', async () => { try { const IDB_KEY = 'indexeddb-delete-needed'; - const item = await sql.sqlCall('getItemById', [IDB_KEY]); + const item = await sql.sqlCall('getItemById', IDB_KEY); if (item && item.value) { - await sql.sqlCall('removeIndexedDBFiles', []); - await sql.sqlCall('removeItemById', [IDB_KEY]); + await sql.sqlCall('removeIndexedDBFiles'); + await sql.sqlCall('removeItemById', IDB_KEY); } } catch (err) { getLogger().error( @@ -1847,43 +1844,6 @@ app.on('ready', async () => { ); } - async function cleanupOrphanedAttachments() { - const allAttachments = await attachments.getAllAttachments(userDataPath); - const orphanedAttachments = await sql.sqlCall('removeKnownAttachments', [ - allAttachments, - ]); - await attachments.deleteAll({ - userDataPath, - attachments: orphanedAttachments, - }); - - await attachments.deleteAllBadges({ - userDataPath, - pathsToKeep: await sql.sqlCall('getAllBadgeImageFileLocalPaths', []), - }); - - const allStickers = await attachments.getAllStickers(userDataPath); - const orphanedStickers = await sql.sqlCall('removeKnownStickers', [ - allStickers, - ]); - await attachments.deleteAllStickers({ - userDataPath, - stickers: orphanedStickers, - }); - - const allDraftAttachments = await attachments.getAllDraftAttachments( - userDataPath - ); - const orphanedDraftAttachments = await sql.sqlCall( - 'removeKnownDraftAttachments', - [allDraftAttachments] - ); - await attachments.deleteAllDraftAttachments({ - userDataPath, - attachments: orphanedDraftAttachments, - }); - } - ready = true; setupMenu(); @@ -2320,10 +2280,7 @@ ipc.on('install-sticker-pack', (_event, packId, packKeyHex) => { } }); -ipc.on('ensure-file-permissions', async event => { - await ensureFilePermissions(); - event.reply('ensure-file-permissions-done'); -}); +ipc.handle('ensure-file-permissions', () => ensureFilePermissions()); /** * Ensure files in the user's data directory have the proper permissions. diff --git a/app/sql_channel.ts b/app/sql_channel.ts index cac43da428..25e442d3d7 100644 --- a/app/sql_channel.ts +++ b/app/sql_channel.ts @@ -3,21 +3,18 @@ import { ipcMain } from 'electron'; +import type { MainSQL } from '../ts/sql/main'; import { remove as removeUserConfig } from './user_config'; import { remove as removeEphemeralConfig } from './ephemeral_config'; -type SQLType = { - sqlCall(callName: string, args: ReadonlyArray): unknown; -}; - -let sql: SQLType | undefined; +let sql: Pick | undefined; let initialized = false; const SQL_CHANNEL_KEY = 'sql-channel'; const ERASE_SQL_KEY = 'erase-sql-key'; -export function initialize(mainSQL: SQLType): void { +export function initialize(mainSQL: Pick): void { if (initialized) { throw new Error('sqlChannels: already initialized!'); } @@ -25,33 +22,15 @@ export function initialize(mainSQL: SQLType): void { sql = mainSQL; - ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => { - try { - if (!sql) { - throw new Error(`${SQL_CHANNEL_KEY}: Not yet initialized!`); - } - const result = await sql.sqlCall(callName, args); - event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log( - `sql channel error with call ${callName}: ${errorForDisplay}` - ); - if (!event.sender.isDestroyed()) { - event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, errorForDisplay); - } + ipcMain.handle(SQL_CHANNEL_KEY, (_event, callName, ...args) => { + if (!sql) { + throw new Error(`${SQL_CHANNEL_KEY}: Not yet initialized!`); } + return sql.sqlCall(callName, ...args); }); - ipcMain.on(ERASE_SQL_KEY, async event => { - try { - removeUserConfig(); - removeEphemeralConfig(); - event.sender.send(`${ERASE_SQL_KEY}-done`); - } catch (error) { - const errorForDisplay = error && error.stack ? error.stack : error; - console.log(`sql-erase error: ${errorForDisplay}`); - event.sender.send(`${ERASE_SQL_KEY}-done`, error); - } + ipcMain.handle(ERASE_SQL_KEY, () => { + removeUserConfig(); + removeEphemeralConfig(); }); } diff --git a/ts/background.ts b/ts/background.ts index 99cac9f900..6b125a41f1 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -849,16 +849,7 @@ export async function startApp(): Promise { ); if (newVersion) { - // We've received reports that this update can take longer than two minutes, so we - // allow it to continue and just move on in that timeout case. - try { - await window.Signal.Data.cleanupOrphanedAttachments(); - } catch (error) { - log.error( - 'background: Failed to cleanup orphaned attachments:', - error && error.stack ? error.stack : error - ); - } + await window.Signal.Data.cleanupOrphanedAttachments(); // Don't block on the following operation window.Signal.Data.ensureFilePermissions(); diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index 3e7eb811ce..94b5be03dd 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -1,8 +1,6 @@ // Copyright 2020-2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -/* eslint-disable no-await-in-loop */ -/* eslint-disable camelcase */ import { ipcRenderer as ipc } from 'electron'; import fs from 'fs-extra'; import pify from 'pify'; @@ -18,28 +16,23 @@ import { map, omit, toPairs, - uniq, } from 'lodash'; import { deleteExternalFiles } from '../types/Conversation'; import { expiringMessagesDeletionService } from '../services/expiringMessagesDeletion'; import { tapToViewMessagesDeletionService } from '../services/tapToViewMessagesDeletionService'; import * as Bytes from '../Bytes'; -import { CURRENT_SCHEMA_VERSION } from '../types/Message2'; import { createBatcher } from '../util/batcher'; +import { explodePromise } from '../util/explodePromise'; import { assertDev, softAssert, strictAssert } from '../util/assert'; import { mapObjectWithSpec } from '../util/mapObjectWithSpec'; import type { ObjectMappingSpecType } from '../util/mapObjectWithSpec'; import { cleanDataForIpc } from './cleanDataForIpc'; -import type { ReactionType } from '../types/Reactions'; -import type { ConversationColorType, CustomColorType } from '../types/Colors'; import type { UUIDStringType } from '../types/UUID'; -import type { BadgeType } from '../badges/types'; -import type { ProcessGroupCallRingRequestResult } from '../types/Calling'; -import type { RemoveAllConfiguration } from '../types/RemoveAllConfiguration'; import createTaskWithTimeout from '../textsecure/TaskWithTimeout'; import * as log from '../logging/log'; import { isValidUuid } from '../types/UUID'; +import * as Errors from '../types/errors'; import type { StoredJob } from '../jobs/types'; import { formatJobForInsert } from '../jobs/formatJobForInsert'; @@ -49,14 +42,9 @@ import type { AllItemsType, AttachmentDownloadJobType, ClientInterface, - ClientJobType, + ClientExclusiveInterface, ClientSearchResultMessageType, ConversationType, - ConversationMetricsType, - DeleteSentProtoRecipientOptionsType, - DeleteSentProtoRecipientResultType, - EmojiType, - GetUnreadByConversationAndMarkReadResultType, GetConversationRangeCenteredOnMessageResultType, IdentityKeyIdType, IdentityKeyType, @@ -64,53 +52,22 @@ import type { ItemKeyType, ItemType, StoredItemType, - ConversationMessageStatsType, MessageType, MessageTypeUnhydrated, PreKeyIdType, PreKeyType, - ReactionResultType, StoredPreKeyType, - SenderKeyIdType, - SenderKeyType, - SentMessageDBType, - SentMessagesType, - SentProtoType, - SentProtoWithMessageIdsType, - SentRecipientsDBType, - SentRecipientsType, ServerInterface, ServerSearchResultMessageType, - SessionIdType, - SessionType, SignedPreKeyIdType, SignedPreKeyType, StoredSignedPreKeyType, - StickerPackStatusType, - StickerPackInfoType, - StickerPackType, - StickerType, - StoryDistributionMemberType, - StoryDistributionType, - StoryDistributionWithMembersType, - StoryReadType, - UnprocessedType, - UnprocessedUpdateType, - UninstalledStickerPackType, } from './Interface'; import Server from './Server'; import { isCorruptionError } from './errors'; import { MINUTE } from '../util/durations'; import { getMessageIdForLogging } from '../util/idForLogging'; -// We listen to a lot of events on ipc, often on the same channel. This prevents -// any warnings that might be sent to the console in that case. -if (ipc && ipc.setMaxListeners) { - ipc.setMaxListeners(0); -} else { - log.warn('sql/Client: ipc is not available!'); -} - const getRealPath = pify(fs.realpath); const MIN_TRACE_DURATION = 10; @@ -124,12 +81,6 @@ const ERASE_DRAFTS_KEY = 'erase-drafts'; const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments'; const ENSURE_FILE_PERMISSIONS = 'ensure-file-permissions'; -type ClientJobUpdateType = { - resolve: (value: unknown) => void; - reject: (error: Error) => void; - args?: ReadonlyArray; -}; - enum RendererState { InMain = 'InMain', Opening = 'Opening', @@ -137,235 +88,13 @@ enum RendererState { Closing = 'Closing', } -const _jobs: { [id: string]: ClientJobType } = Object.create(null); -const _DEBUG = false; -let _jobCounter = 0; -let _shuttingDown = false; -let _shutdownCallback: ((error?: Error) => void) | null = null; -let _shutdownPromise: Promise | null = null; +let activeJobCount = 0; +let resolveShutdown: (() => void) | undefined; +let shutdownPromise: Promise | null = null; + let state = RendererState.InMain; const startupQueries = new Map(); -// Because we can't force this module to conform to an interface, we narrow our exports -// to this one default export, which does conform to the interface. -// Note: In Javascript, you need to access the .default property when requiring it -// https://github.com/microsoft/TypeScript/issues/420 -const dataInterface: ClientInterface = { - close, - removeDB, - removeIndexedDBFiles, - - createOrUpdateIdentityKey, - getIdentityKeyById, - bulkAddIdentityKeys, - removeIdentityKeyById, - removeAllIdentityKeys, - getAllIdentityKeys, - - createOrUpdatePreKey, - getPreKeyById, - bulkAddPreKeys, - removePreKeyById, - removePreKeysByUuid, - removeAllPreKeys, - getAllPreKeys, - - createOrUpdateSignedPreKey, - getSignedPreKeyById, - bulkAddSignedPreKeys, - removeSignedPreKeyById, - removeSignedPreKeysByUuid, - removeAllSignedPreKeys, - getAllSignedPreKeys, - - createOrUpdateItem, - getItemById, - removeItemById, - removeAllItems, - getAllItems, - - createOrUpdateSenderKey, - getSenderKeyById, - removeAllSenderKeys, - getAllSenderKeys, - removeSenderKeyById, - - insertSentProto, - deleteSentProtosOlderThan, - deleteSentProtoByMessageId, - insertProtoRecipients, - deleteSentProtoRecipient, - getSentProtoByRecipient, - removeAllSentProtos, - getAllSentProtos, - _getAllSentProtoRecipients, - _getAllSentProtoMessageIds, - - createOrUpdateSession, - createOrUpdateSessions, - commitDecryptResult, - bulkAddSessions, - removeSessionById, - removeSessionsByConversation, - removeAllSessions, - getAllSessions, - - eraseStorageServiceStateFromConversations, - getConversationCount, - saveConversation, - saveConversations, - getConversationById, - updateConversation, - updateConversations, - removeConversation, - _removeAllConversations, - updateAllConversationColors, - removeAllProfileKeyCredentials, - - getAllConversations, - getAllConversationIds, - getAllGroupsInvolvingUuid, - - searchMessages, - searchMessagesInConversation, - - getMessageCount, - getStoryCount, - saveMessage, - saveMessages, - removeMessage, - removeMessages, - getTotalUnreadForConversation, - getUnreadByConversationAndMarkRead, - getUnreadReactionsAndMarkRead, - markReactionAsRead, - removeReactionFromConversation, - addReaction, - _getAllReactions, - _removeAllReactions, - getMessageBySender, - getMessageById, - getMessagesById, - _getAllMessages, - _removeAllMessages, - getAllMessageIds, - getMessagesBySentAt, - getExpiredMessages, - getMessagesUnexpectedlyMissingExpirationStartTimestamp, - getSoonestMessageExpiry, - getNextTapToViewMessageTimestampToAgeOut, - getTapToViewMessagesNeedingErase, - getOlderMessagesByConversation, - getAllStories, - hasStoryReplies, - hasStoryRepliesFromSelf, - getNewerMessagesByConversation, - getMessageMetricsForConversation, - getConversationRangeCenteredOnMessage, - getConversationMessageStats, - getLastConversationMessage, - hasGroupCallHistoryMessage, - migrateConversationMessages, - - getUnprocessedCount, - getAllUnprocessedAndIncrementAttempts, - getUnprocessedById, - updateUnprocessedWithData, - updateUnprocessedsWithData, - removeUnprocessed, - removeAllUnprocessed, - - getAttachmentDownloadJobById, - getNextAttachmentDownloadJobs, - saveAttachmentDownloadJob, - resetAttachmentDownloadPending, - setAttachmentDownloadJobPending, - removeAttachmentDownloadJob, - removeAllAttachmentDownloadJobs, - - createOrUpdateStickerPack, - updateStickerPackStatus, - updateStickerPackInfo, - createOrUpdateSticker, - updateStickerLastUsed, - addStickerPackReference, - deleteStickerPackReference, - getStickerCount, - deleteStickerPack, - getAllStickerPacks, - addUninstalledStickerPack, - removeUninstalledStickerPack, - getInstalledStickerPacks, - getUninstalledStickerPacks, - installStickerPack, - uninstallStickerPack, - getStickerPackInfo, - getAllStickers, - getRecentStickers, - clearAllErrorStickerPackAttempts, - - updateEmojiUsage, - getRecentEmojis, - - getAllBadges, - updateOrCreateBadges, - badgeImageFileDownloaded, - - _getAllStoryDistributions, - _getAllStoryDistributionMembers, - _deleteAllStoryDistributions, - createNewStoryDistribution, - getAllStoryDistributionsWithMembers, - getStoryDistributionWithMembers, - modifyStoryDistribution, - modifyStoryDistributionMembers, - modifyStoryDistributionWithMembers, - deleteStoryDistribution, - - _getAllStoryReads, - _deleteAllStoryReads, - addNewStoryRead, - getLastStoryReadsForAuthor, - countStoryReadsByConversation, - - removeAll, - removeAllConfiguration, - - getMessagesNeedingUpgrade, - getMessagesWithVisualMediaAttachments, - getMessagesWithFileAttachments, - getMessageServerGuidsForSpam, - - getJobsInQueue, - insertJob, - deleteJob, - - processGroupCallRingRequest, - processGroupCallRingCancelation, - cleanExpiredGroupCallRings, - - getMaxMessageCounter, - - getStatisticsForLogging, - - // Client-side only - - shutdown, - removeAllMessagesInConversation, - - removeOtherData, - cleanupOrphanedAttachments, - ensureFilePermissions, - - // Client-side only, and test-only - - startInRendererProcess, - goBackToMainProcess, - _jobs, -}; - -export default dataInterface; - async function startInRendererProcess(isTesting = false): Promise { strictAssert( state === RendererState.InMain, @@ -376,13 +105,7 @@ async function startInRendererProcess(isTesting = false): Promise { state = RendererState.Opening; if (!isTesting) { - ipc.send('database-ready'); - - await new Promise(resolve => { - ipc.once('database-ready', () => { - resolve(); - }); - }); + await ipc.invoke('database-ready'); } const configDir = await getRealPath(ipc.sendSync('get-user-data-path')); @@ -408,7 +131,7 @@ async function goBackToMainProcess(): Promise { // We don't need to wait for pending queries since they are synchronous. log.info('data.goBackToMainProcess: switching to main process'); - const closePromise = close(); + const closePromise = channels.close(); // It should be the last query we run in renderer process state = RendererState.Closing; @@ -432,7 +155,7 @@ async function goBackToMainProcess(): Promise { const channelsAsUnknown = fromPairs( compact( - map(toPairs(dataInterface), ([name, value]: [string, unknown]) => { + map(toPairs(Server), ([name, value]: [string, unknown]) => { if (isFunction(value)) { return [name, makeChannel(name)]; } @@ -444,6 +167,69 @@ const channelsAsUnknown = fromPairs( const channels: ServerInterface = channelsAsUnknown as ServerInterface; +const exclusiveInterface: ClientExclusiveInterface = { + createOrUpdateIdentityKey, + getIdentityKeyById, + bulkAddIdentityKeys, + getAllIdentityKeys, + + createOrUpdatePreKey, + getPreKeyById, + bulkAddPreKeys, + getAllPreKeys, + + createOrUpdateSignedPreKey, + getSignedPreKeyById, + bulkAddSignedPreKeys, + getAllSignedPreKeys, + + createOrUpdateItem, + getItemById, + getAllItems, + + updateConversation, + removeConversation, + + searchMessages, + searchMessagesInConversation, + + getOlderMessagesByConversation, + getConversationRangeCenteredOnMessage, + getNewerMessagesByConversation, + + // Client-side only + + shutdown, + removeAllMessagesInConversation, + + removeOtherData, + cleanupOrphanedAttachments, + ensureFilePermissions, + + // Client-side only, and test-only + + startInRendererProcess, + goBackToMainProcess, +}; + +// Because we can't force this module to conform to an interface, we narrow our exports +// to this one default export, which does conform to the interface. +// Note: In Javascript, you need to access the .default property when requiring it +// https://github.com/microsoft/TypeScript/issues/420 +const dataInterface: ClientInterface = { + ...channels, + ...exclusiveInterface, + + // Overrides + updateConversations, + saveMessage, + saveMessages, + removeMessage, + saveAttachmentDownloadJob, +}; + +export default dataInterface; + function _cleanData( data: unknown ): ReturnType['cleaned'] { @@ -481,141 +267,28 @@ export function _cleanMessageData(data: MessageType): MessageType { return _cleanData(omit(result, ['dataMessage'])); } -async function _shutdown() { - const jobKeys = Object.keys(_jobs); +async function doShutdown() { log.info( - `data.shutdown: shutdown requested. ${jobKeys.length} jobs outstanding` + `data.shutdown: shutdown requested. ${activeJobCount} jobs outstanding` ); - if (_shutdownPromise) { - await _shutdownPromise; - - return; + if (shutdownPromise) { + return shutdownPromise; } - _shuttingDown = true; - // No outstanding jobs, return immediately - if (jobKeys.length === 0 || _DEBUG) { + if (activeJobCount === 0) { return; } - // Outstanding jobs; we need to wait until the last one is done - _shutdownPromise = new Promise((resolve, reject) => { - _shutdownCallback = (error?: Error) => { - log.info('data.shutdown: process complete'); - if (error) { - reject(error); + ({ promise: shutdownPromise, resolve: resolveShutdown } = + explodePromise()); - return; - } - - resolve(); - }; - }); - - await _shutdownPromise; -} - -function _makeJob(fnName: string) { - if (_shuttingDown && fnName !== 'close') { - throw new Error( - `Rejecting SQL channel job (${fnName}); application is shutting down` - ); + try { + await shutdownPromise; + } finally { + log.info('data.shutdown: process complete'); } - - _jobCounter += 1; - const id = _jobCounter; - - if (_DEBUG) { - log.info(`SQL channel job ${id} (${fnName}) started`); - } - _jobs[id] = { - fnName, - start: Date.now(), - }; - - return id; -} - -function _updateJob(id: number, data: ClientJobUpdateType) { - const { resolve, reject } = data; - const { fnName, start } = _jobs[id]; - - _jobs[id] = { - ..._jobs[id], - ...data, - resolve: (value: unknown) => { - _removeJob(id); - const end = Date.now(); - if (_DEBUG) { - log.info( - `SQL channel job ${id} (${fnName}) succeeded in ${end - start}ms` - ); - } - - return resolve(value); - }, - reject: (error: Error) => { - _removeJob(id); - const end = Date.now(); - log.info(`SQL channel job ${id} (${fnName}) failed in ${end - start}ms`); - - return reject(error); - }, - }; -} - -function _removeJob(id: number) { - if (_DEBUG) { - _jobs[id].complete = true; - - return; - } - - delete _jobs[id]; - - if (_shutdownCallback) { - const keys = Object.keys(_jobs); - if (keys.length === 0) { - _shutdownCallback(); - } - } -} - -function _getJob(id: number) { - return _jobs[id]; -} - -if (ipc && ipc.on) { - ipc.on(`${SQL_CHANNEL_KEY}-done`, (_, jobId, errorForDisplay, result) => { - const job = _getJob(jobId); - if (!job) { - throw new Error( - `Received SQL channel reply to job ${jobId}, but did not have it in our registry!` - ); - } - - const { resolve, reject, fnName } = job; - - if (!resolve || !reject) { - throw new Error( - `SQL channel job ${jobId} (${fnName}): didn't have a resolve or reject` - ); - } - - if (errorForDisplay) { - return reject( - new Error( - `Error received from SQL channel job ${jobId} (${fnName}): ${errorForDisplay}` - ) - ); - } - - return resolve(result); - }); -} else { - log.warn('sql/Client: ipc.on is not available!'); } function makeChannel(fnName: string) { @@ -654,7 +327,7 @@ function makeChannel(fnName: string) { (startupQueries.get(serverFnName) || 0) + duration ); - if (duration > MIN_TRACE_DURATION || _DEBUG) { + if (duration > MIN_TRACE_DURATION) { log.info( `Renderer SQL channel job (${fnName}) completed in ${duration}ms` ); @@ -662,27 +335,23 @@ function makeChannel(fnName: string) { } } - const jobId = _makeJob(fnName); + if (shutdownPromise && fnName !== 'close') { + throw new Error( + `Rejecting SQL channel job (${fnName}); application is shutting down` + ); + } - return createTaskWithTimeout( - () => - new Promise((resolve, reject) => { - try { - ipc.send(SQL_CHANNEL_KEY, jobId, fnName, ...args); - - _updateJob(jobId, { - resolve, - reject, - args: _DEBUG ? args : undefined, - }); - } catch (error) { - _removeJob(jobId); - - reject(error); - } - }), - `SQL channel job ${jobId} (${fnName})` - )(); + activeJobCount += 1; + return createTaskWithTimeout(async () => { + try { + return await ipc.invoke(SQL_CHANNEL_KEY, fnName, ...args); + } finally { + activeJobCount -= 1; + if (activeJobCount === 0) { + resolveShutdown?.(); + } + } + }, `SQL channel call (${fnName})`)(); }; } @@ -710,26 +379,12 @@ async function shutdown(): Promise { log.info('Client.shutdown'); // Stop accepting new SQL jobs, flush outstanding queue - await _shutdown(); + await doShutdown(); // Close database - await close(); -} - -// Note: will need to restart the app after calling this, to set up afresh -async function close(): Promise { await channels.close(); } -// Note: will need to restart the app after calling this, to set up afresh -async function removeDB(): Promise { - await channels.removeDB(); -} - -async function removeIndexedDBFiles(): Promise { - await channels.removeIndexedDBFiles(); -} - // Identity Keys const IDENTITY_KEY_SPEC = ['publicKey']; @@ -752,12 +407,6 @@ async function bulkAddIdentityKeys( ); await channels.bulkAddIdentityKeys(updated); } -async function removeIdentityKeyById(id: IdentityKeyIdType): Promise { - await channels.removeIdentityKeyById(id); -} -async function removeAllIdentityKeys(): Promise { - await channels.removeAllIdentityKeys(); -} async function getAllIdentityKeys(): Promise> { const keys = await channels.getAllIdentityKeys(); @@ -783,15 +432,6 @@ async function bulkAddPreKeys(array: Array): Promise { ); await channels.bulkAddPreKeys(updated); } -async function removePreKeyById(id: PreKeyIdType): Promise { - await channels.removePreKeyById(id); -} -async function removePreKeysByUuid(uuid: UUIDStringType): Promise { - await channels.removePreKeysByUuid(uuid); -} -async function removeAllPreKeys(): Promise { - await channels.removeAllPreKeys(); -} async function getAllPreKeys(): Promise> { const keys = await channels.getAllPreKeys(); @@ -827,15 +467,6 @@ async function bulkAddSignedPreKeys( ); await channels.bulkAddSignedPreKeys(updated); } -async function removeSignedPreKeyById(id: SignedPreKeyIdType): Promise { - await channels.removeSignedPreKeyById(id); -} -async function removeSignedPreKeysByUuid(uuid: UUIDStringType): Promise { - await channels.removeSignedPreKeysByUuid(uuid); -} -async function removeAllSignedPreKeys(): Promise { - await channels.removeAllSignedPreKeys(); -} // Items @@ -897,153 +528,9 @@ async function getAllItems(): Promise { return result; } -async function removeItemById(id: ItemKeyType): Promise { - await channels.removeItemById(id); -} -async function removeAllItems(): Promise { - await channels.removeAllItems(); -} - -// Sender Keys - -async function createOrUpdateSenderKey(key: SenderKeyType): Promise { - await channels.createOrUpdateSenderKey(key); -} -async function getSenderKeyById( - id: SenderKeyIdType -): Promise { - return channels.getSenderKeyById(id); -} -async function removeAllSenderKeys(): Promise { - await channels.removeAllSenderKeys(); -} -async function getAllSenderKeys(): Promise> { - return channels.getAllSenderKeys(); -} -async function removeSenderKeyById(id: SenderKeyIdType): Promise { - return channels.removeSenderKeyById(id); -} - -// Sent Protos - -async function insertSentProto( - proto: SentProtoType, - options: { - messageIds: SentMessagesType; - recipients: SentRecipientsType; - } -): Promise { - return channels.insertSentProto(proto, { - ...options, - messageIds: uniq(options.messageIds), - }); -} -async function deleteSentProtosOlderThan(timestamp: number): Promise { - await channels.deleteSentProtosOlderThan(timestamp); -} -async function deleteSentProtoByMessageId(messageId: string): Promise { - await channels.deleteSentProtoByMessageId(messageId); -} - -async function insertProtoRecipients(options: { - id: number; - recipientUuid: string; - deviceIds: Array; -}): Promise { - await channels.insertProtoRecipients(options); -} -async function deleteSentProtoRecipient( - options: - | DeleteSentProtoRecipientOptionsType - | ReadonlyArray -): Promise { - return channels.deleteSentProtoRecipient(options); -} - -async function getSentProtoByRecipient(options: { - now: number; - recipientUuid: string; - timestamp: number; -}): Promise { - return channels.getSentProtoByRecipient(options); -} -async function removeAllSentProtos(): Promise { - await channels.removeAllSentProtos(); -} -async function getAllSentProtos(): Promise> { - return channels.getAllSentProtos(); -} - -// Test-only: -async function _getAllSentProtoRecipients(): Promise< - Array -> { - return channels._getAllSentProtoRecipients(); -} -async function _getAllSentProtoMessageIds(): Promise> { - return channels._getAllSentProtoMessageIds(); -} - -// Sessions - -async function createOrUpdateSession(data: SessionType): Promise { - await channels.createOrUpdateSession(data); -} -async function createOrUpdateSessions( - array: Array -): Promise { - await channels.createOrUpdateSessions(array); -} -async function commitDecryptResult(options: { - senderKeys: Array; - sessions: Array; - unprocessed: Array; -}): Promise { - await channels.commitDecryptResult(options); -} -async function bulkAddSessions(array: Array): Promise { - await channels.bulkAddSessions(array); -} -async function removeSessionById(id: SessionIdType): Promise { - await channels.removeSessionById(id); -} - -async function removeSessionsByConversation( - conversationId: string -): Promise { - await channels.removeSessionsByConversation(conversationId); -} -async function removeAllSessions(): Promise { - await channels.removeAllSessions(); -} -async function getAllSessions(): Promise> { - const sessions = await channels.getAllSessions(); - - return sessions; -} // Conversation -async function getConversationCount(): Promise { - return channels.getConversationCount(); -} - -async function saveConversation(data: ConversationType): Promise { - await channels.saveConversation(data); -} - -async function saveConversations( - array: Array -): Promise { - await channels.saveConversations(array); -} - -async function getConversationById( - id: string -): Promise { - return channels.getConversationById(id); -} - const updateConversationBatcher = createBatcher({ name: 'sql.Client.updateConversationBatcher', wait: 500, @@ -1078,7 +565,7 @@ async function updateConversations( } async function removeConversation(id: string): Promise { - const existing = await getConversationById(id); + const existing = await channels.getConversationById(id); // Note: It's important to have a fully database-hydrated model to delete here because // it needs to delete all associated on-disk files along with the database delete. @@ -1090,30 +577,6 @@ async function removeConversation(id: string): Promise { } } -async function _removeAllConversations(): Promise { - await channels._removeAllConversations(); -} - -async function eraseStorageServiceStateFromConversations(): Promise { - await channels.eraseStorageServiceStateFromConversations(); -} - -async function getAllConversations(): Promise> { - return channels.getAllConversations(); -} - -async function getAllConversationIds(): Promise> { - const ids = await channels.getAllConversationIds(); - - return ids; -} - -async function getAllGroupsInvolvingUuid( - uuid: UUIDStringType -): Promise> { - return channels.getAllGroupsInvolvingUuid(uuid); -} - function handleSearchMessageJSON( messages: Array ): Array { @@ -1153,14 +616,6 @@ async function searchMessagesInConversation( // Message -async function getMessageCount(conversationId?: string): Promise { - return channels.getMessageCount(conversationId); -} - -async function getStoryCount(conversationId: string): Promise { - return channels.getStoryCount(conversationId); -} - async function saveMessage( data: MessageType, options: { @@ -1196,7 +651,7 @@ async function saveMessages( } async function removeMessage(id: string): Promise { - const message = await getMessageById(id); + const message = await channels.getMessageById(id); // Note: It's important to have a fully database-hydrated model to delete here because // it needs to delete all associated on-disk files along with the database delete. @@ -1206,119 +661,42 @@ async function removeMessage(id: string): Promise { } } -// Note: this method will not clean up external files, just delete from SQL -async function removeMessages(ids: Array): Promise { - await channels.removeMessages(ids); -} - -async function getMessageById(id: string): Promise { - return channels.getMessageById(id); -} - -async function getMessagesById( - messageIds: Array -): Promise> { - if (!messageIds.length) { - return []; - } - return channels.getMessagesById(messageIds); -} - -// For testing only -async function _getAllMessages(): Promise> { - return channels._getAllMessages(); -} -async function _removeAllMessages(): Promise { - await channels._removeAllMessages(); -} - -async function getAllMessageIds(): Promise> { - const ids = await channels.getAllMessageIds(); - - return ids; -} - -async function getMessageBySender({ - source, - sourceUuid, - sourceDevice, - sent_at, -}: { - source: string; - sourceUuid: UUIDStringType; - sourceDevice: number; - sent_at: number; -}): Promise { - return channels.getMessageBySender({ - source, - sourceUuid, - sourceDevice, - sent_at, - }); -} - -async function getTotalUnreadForConversation( - conversationId: string, - options: { - storyId: UUIDStringType | undefined; - includeStoryReplies: boolean; - } -): Promise { - return channels.getTotalUnreadForConversation(conversationId, options); -} - -async function getUnreadByConversationAndMarkRead(options: { - conversationId: string; - includeStoryReplies: boolean; - newestUnreadAt: number; - now?: number; - readAt?: number; - storyId?: UUIDStringType; -}): Promise { - return channels.getUnreadByConversationAndMarkRead(options); -} - -async function getUnreadReactionsAndMarkRead(options: { - conversationId: string; - newestUnreadAt: number; - storyId?: UUIDStringType; -}): Promise> { - return channels.getUnreadReactionsAndMarkRead(options); -} - -async function markReactionAsRead( - targetAuthorUuid: string, - targetTimestamp: number -): Promise { - return channels.markReactionAsRead(targetAuthorUuid, targetTimestamp); -} - -async function removeReactionFromConversation(reaction: { - emoji: string; - fromId: string; - targetAuthorUuid: string; - targetTimestamp: number; -}): Promise { - return channels.removeReactionFromConversation(reaction); -} - -async function addReaction(reactionObj: ReactionType): Promise { - return channels.addReaction(reactionObj); -} - -async function _getAllReactions(): Promise> { - return channels._getAllReactions(); -} -async function _removeAllReactions(): Promise { - await channels._removeAllReactions(); -} - function handleMessageJSON( messages: Array ): Array { return messages.map(message => JSON.parse(message.json)); } +async function getNewerMessagesByConversation( + conversationId: string, + { + includeStoryReplies, + limit = 100, + receivedAt = 0, + sentAt = 0, + storyId, + }: { + includeStoryReplies: boolean; + limit?: number; + receivedAt?: number; + sentAt?: number; + storyId: UUIDStringType | undefined; + } +): Promise> { + const messages = await channels.getNewerMessagesByConversation( + conversationId, + { + includeStoryReplies, + limit, + receivedAt, + sentAt, + storyId, + } + ); + + return handleMessageJSON(messages); +} + async function getOlderMessagesByConversation( conversationId: string, { @@ -1352,93 +730,6 @@ async function getOlderMessagesByConversation( return handleMessageJSON(messages); } -async function getAllStories(options: { - conversationId?: string; - sourceUuid?: UUIDStringType; -}): Promise> { - return channels.getAllStories(options); -} - -async function hasStoryReplies(storyId: string): Promise { - return channels.hasStoryReplies(storyId); -} - -async function hasStoryRepliesFromSelf(storyId: string): Promise { - return channels.hasStoryRepliesFromSelf(storyId); -} - -async function getNewerMessagesByConversation( - conversationId: string, - { - includeStoryReplies, - limit = 100, - receivedAt = 0, - sentAt = 0, - storyId, - }: { - includeStoryReplies: boolean; - limit?: number; - receivedAt?: number; - sentAt?: number; - storyId: UUIDStringType | undefined; - } -): Promise> { - const messages = await channels.getNewerMessagesByConversation( - conversationId, - { - includeStoryReplies, - limit, - receivedAt, - sentAt, - storyId, - } - ); - - return handleMessageJSON(messages); -} -async function getConversationMessageStats({ - conversationId, - includeStoryReplies, - ourUuid, -}: { - conversationId: string; - includeStoryReplies: boolean; - ourUuid: UUIDStringType; -}): Promise { - const { preview, activity, hasUserInitiatedMessages } = - await channels.getConversationMessageStats({ - conversationId, - includeStoryReplies, - ourUuid, - }); - - return { - preview, - activity, - hasUserInitiatedMessages, - }; -} -async function getLastConversationMessage({ - conversationId, -}: { - conversationId: string; -}): Promise { - return channels.getLastConversationMessage({ conversationId }); -} -async function getMessageMetricsForConversation( - conversationId: string, - options: { - storyId?: UUIDStringType; - includeStoryReplies: boolean; - } -): Promise { - const result = await channels.getMessageMetricsForConversation( - conversationId, - options - ); - - return result; -} async function getConversationRangeCenteredOnMessage(options: { conversationId: string; includeStoryReplies: boolean; @@ -1457,19 +748,6 @@ async function getConversationRangeCenteredOnMessage(options: { }; } -function hasGroupCallHistoryMessage( - conversationId: string, - eraId: string -): Promise { - return channels.hasGroupCallHistoryMessage(conversationId, eraId); -} -async function migrateConversationMessages( - obsoleteId: string, - currentId: string -): Promise { - await channels.migrateConversationMessages(obsoleteId, currentId); -} - async function removeAllMessagesInConversation( conversationId: string, { @@ -1486,6 +764,7 @@ async function removeAllMessagesInConversation( ); // Yes, we really want the await in the loop. We're deleting a chunk at a // time so we don't use too much memory. + // eslint-disable-next-line no-await-in-loop messages = await getOlderMessagesByConversation(conversationId, { limit: chunkSize, includeStoryReplies: true, @@ -1507,478 +786,54 @@ async function removeAllMessagesInConversation( (message: MessageType) => async () => cleanupMessage(message) ) ); + // eslint-disable-next-line no-await-in-loop await queue.onIdle(); log.info(`removeAllMessagesInConversation/${logId}: Deleting...`); + // eslint-disable-next-line no-await-in-loop await channels.removeMessages(ids); } while (messages.length > 0); } -async function getMessagesBySentAt( - sentAt: number -): Promise> { - return channels.getMessagesBySentAt(sentAt); -} - -async function getExpiredMessages(): Promise> { - return channels.getExpiredMessages(); -} - -function getMessagesUnexpectedlyMissingExpirationStartTimestamp(): Promise< - Array -> { - return channels.getMessagesUnexpectedlyMissingExpirationStartTimestamp(); -} - -function getSoonestMessageExpiry(): Promise { - return channels.getSoonestMessageExpiry(); -} - -async function getNextTapToViewMessageTimestampToAgeOut(): Promise< - number | undefined -> { - return channels.getNextTapToViewMessageTimestampToAgeOut(); -} -async function getTapToViewMessagesNeedingErase(): Promise> { - return channels.getTapToViewMessagesNeedingErase(); -} - -// Unprocessed - -async function getUnprocessedCount(): Promise { - return channels.getUnprocessedCount(); -} - -async function getAllUnprocessedAndIncrementAttempts(): Promise< - Array -> { - return channels.getAllUnprocessedAndIncrementAttempts(); -} - -async function getUnprocessedById( - id: string -): Promise { - return channels.getUnprocessedById(id); -} - -async function updateUnprocessedWithData( - id: string, - data: UnprocessedUpdateType -): Promise { - await channels.updateUnprocessedWithData(id, data); -} -async function updateUnprocessedsWithData( - array: Array<{ id: string; data: UnprocessedUpdateType }> -): Promise { - await channels.updateUnprocessedsWithData(array); -} - -async function removeUnprocessed(id: string | Array): Promise { - await channels.removeUnprocessed(id); -} - -async function removeAllUnprocessed(): Promise { - await channels.removeAllUnprocessed(); -} - // Attachment downloads -async function getAttachmentDownloadJobById( - id: string -): Promise { - return channels.getAttachmentDownloadJobById(id); -} -async function getNextAttachmentDownloadJobs( - limit?: number, - options?: { timestamp?: number } -): Promise> { - return channels.getNextAttachmentDownloadJobs(limit, options); -} async function saveAttachmentDownloadJob( job: AttachmentDownloadJobType ): Promise { await channels.saveAttachmentDownloadJob(_cleanData(job)); } -async function setAttachmentDownloadJobPending( - id: string, - pending: boolean -): Promise { - await channels.setAttachmentDownloadJobPending(id, pending); -} -async function resetAttachmentDownloadPending(): Promise { - await channels.resetAttachmentDownloadPending(); -} -async function removeAttachmentDownloadJob(id: string): Promise { - await channels.removeAttachmentDownloadJob(id); -} -async function removeAllAttachmentDownloadJobs(): Promise { - await channels.removeAllAttachmentDownloadJobs(); -} - -// Stickers - -async function getStickerCount(): Promise { - return channels.getStickerCount(); -} - -async function createOrUpdateStickerPack(pack: StickerPackType): Promise { - await channels.createOrUpdateStickerPack(pack); -} -async function updateStickerPackStatus( - packId: string, - status: StickerPackStatusType, - options?: { timestamp: number } -): Promise { - await channels.updateStickerPackStatus(packId, status, options); -} -async function updateStickerPackInfo(info: StickerPackInfoType): Promise { - await channels.updateStickerPackInfo(info); -} -async function createOrUpdateSticker(sticker: StickerType): Promise { - await channels.createOrUpdateSticker(sticker); -} -async function updateStickerLastUsed( - packId: string, - stickerId: number, - timestamp: number -): Promise { - return channels.updateStickerLastUsed(packId, stickerId, timestamp); -} -async function addStickerPackReference( - messageId: string, - packId: string -): Promise { - await channels.addStickerPackReference(messageId, packId); -} -async function deleteStickerPackReference( - messageId: string, - packId: string -): Promise | undefined> { - return channels.deleteStickerPackReference(messageId, packId); -} -async function deleteStickerPack(packId: string): Promise> { - return channels.deleteStickerPack(packId); -} -async function getAllStickerPacks(): Promise> { - const packs = await channels.getAllStickerPacks(); - - return packs; -} -async function addUninstalledStickerPack( - pack: UninstalledStickerPackType -): Promise { - return channels.addUninstalledStickerPack(pack); -} -async function removeUninstalledStickerPack(packId: string): Promise { - return channels.removeUninstalledStickerPack(packId); -} -async function getInstalledStickerPacks(): Promise> { - return channels.getInstalledStickerPacks(); -} -async function getUninstalledStickerPacks(): Promise< - Array -> { - return channels.getUninstalledStickerPacks(); -} -async function installStickerPack( - packId: string, - timestamp: number -): Promise { - return channels.installStickerPack(packId, timestamp); -} -async function uninstallStickerPack( - packId: string, - timestamp: number -): Promise { - return channels.uninstallStickerPack(packId, timestamp); -} -async function getStickerPackInfo( - packId: string -): Promise { - return channels.getStickerPackInfo(packId); -} -async function getAllStickers(): Promise> { - const stickers = await channels.getAllStickers(); - - return stickers; -} -async function getRecentStickers(): Promise> { - const recentStickers = await channels.getRecentStickers(); - - return recentStickers; -} -async function clearAllErrorStickerPackAttempts(): Promise { - await channels.clearAllErrorStickerPackAttempts(); -} - -// Emojis -async function updateEmojiUsage(shortName: string): Promise { - await channels.updateEmojiUsage(shortName); -} -async function getRecentEmojis(limit = 32): Promise> { - return channels.getRecentEmojis(limit); -} - -// Badges - -function getAllBadges(): Promise> { - return channels.getAllBadges(); -} - -async function updateOrCreateBadges( - badges: ReadonlyArray -): Promise { - if (badges.length) { - await channels.updateOrCreateBadges(badges); - } -} - -function badgeImageFileDownloaded( - url: string, - localPath: string -): Promise { - return channels.badgeImageFileDownloaded(url, localPath); -} - -// Story Distributions - -async function _getAllStoryDistributions(): Promise< - Array -> { - return channels._getAllStoryDistributions(); -} -async function _getAllStoryDistributionMembers(): Promise< - Array -> { - return channels._getAllStoryDistributionMembers(); -} -async function _deleteAllStoryDistributions(): Promise { - await channels._deleteAllStoryDistributions(); -} -async function createNewStoryDistribution( - distribution: StoryDistributionWithMembersType -): Promise { - strictAssert( - distribution.name, - 'Distribution list does not have a valid name' - ); - await channels.createNewStoryDistribution(distribution); -} -async function getAllStoryDistributionsWithMembers(): Promise< - Array -> { - return channels.getAllStoryDistributionsWithMembers(); -} -async function getStoryDistributionWithMembers( - id: string -): Promise { - return channels.getStoryDistributionWithMembers(id); -} -async function modifyStoryDistribution( - distribution: StoryDistributionType -): Promise { - if (distribution.deletedAtTimestamp) { - strictAssert( - !distribution.name, - 'Attempt to delete distribution list but still has a name' - ); - } else { - strictAssert( - distribution.name, - 'Cannot clear distribution list name without deletedAtTimestamp set' - ); - } - await channels.modifyStoryDistribution(distribution); -} -async function modifyStoryDistributionMembers( - id: string, - options: { - toAdd: Array; - toRemove: Array; - } -): Promise { - await channels.modifyStoryDistributionMembers(id, options); -} -async function modifyStoryDistributionWithMembers( - distribution: StoryDistributionType, - options: { - toAdd: Array; - toRemove: Array; - } -): Promise { - if (distribution.deletedAtTimestamp) { - strictAssert( - !distribution.name, - 'Attempt to delete distribution list but still has a name' - ); - } else { - strictAssert( - distribution.name, - 'Cannot clear distribution list name without deletedAtTimestamp set' - ); - } - await channels.modifyStoryDistributionWithMembers(distribution, options); -} -async function deleteStoryDistribution(id: UUIDStringType): Promise { - await channels.deleteStoryDistribution(id); -} - -// Story Reads - -async function _getAllStoryReads(): Promise> { - return channels._getAllStoryReads(); -} -async function _deleteAllStoryReads(): Promise { - await channels._deleteAllStoryReads(); -} -async function addNewStoryRead(read: StoryReadType): Promise { - return channels.addNewStoryRead(read); -} -async function getLastStoryReadsForAuthor(options: { - authorId: UUIDStringType; - conversationId?: UUIDStringType; - limit?: number; -}): Promise> { - return channels.getLastStoryReadsForAuthor(options); -} -async function countStoryReadsByConversation( - conversationId: string -): Promise { - return channels.countStoryReadsByConversation(conversationId); -} // Other -async function removeAll(): Promise { - await channels.removeAll(); -} - -async function removeAllConfiguration( - type?: RemoveAllConfiguration -): Promise { - await channels.removeAllConfiguration(type); -} - async function cleanupOrphanedAttachments(): Promise { - await callChannel(CLEANUP_ORPHANED_ATTACHMENTS_KEY); + try { + await invokeWithTimeout(CLEANUP_ORPHANED_ATTACHMENTS_KEY); + } catch (error) { + log.warn( + 'sql/Client: cleanupOrphanedAttachments failure', + Errors.toLogFormat(error) + ); + } } async function ensureFilePermissions(): Promise { - await callChannel(ENSURE_FILE_PERMISSIONS); + await invokeWithTimeout(ENSURE_FILE_PERMISSIONS); } // Note: will need to restart the app after calling this, to set up afresh async function removeOtherData(): Promise { await Promise.all([ - callChannel(ERASE_SQL_KEY), - callChannel(ERASE_ATTACHMENTS_KEY), - callChannel(ERASE_STICKERS_KEY), - callChannel(ERASE_TEMP_KEY), - callChannel(ERASE_DRAFTS_KEY), + invokeWithTimeout(ERASE_SQL_KEY), + invokeWithTimeout(ERASE_ATTACHMENTS_KEY), + invokeWithTimeout(ERASE_STICKERS_KEY), + invokeWithTimeout(ERASE_TEMP_KEY), + invokeWithTimeout(ERASE_DRAFTS_KEY), ]); } -async function callChannel(name: string): Promise { +async function invokeWithTimeout(name: string): Promise { return createTaskWithTimeout( - () => - new Promise((resolve, reject) => { - ipc.send(name); - ipc.once(`${name}-done`, (_, error) => { - if (error) { - reject(error); - - return; - } - - resolve(); - }); - }), + () => ipc.invoke(name), `callChannel call to ${name}` )(); } - -async function getMessagesNeedingUpgrade( - limit: number, - { maxVersion = CURRENT_SCHEMA_VERSION }: { maxVersion: number } -): Promise> { - const messages = await channels.getMessagesNeedingUpgrade(limit, { - maxVersion, - }); - - return messages; -} - -async function getMessagesWithVisualMediaAttachments( - conversationId: string, - { limit }: { limit: number } -): Promise> { - return channels.getMessagesWithVisualMediaAttachments(conversationId, { - limit, - }); -} - -async function getMessagesWithFileAttachments( - conversationId: string, - { limit }: { limit: number } -): Promise> { - return channels.getMessagesWithFileAttachments(conversationId, { - limit, - }); -} - -function getMessageServerGuidsForSpam( - conversationId: string -): Promise> { - return channels.getMessageServerGuidsForSpam(conversationId); -} - -function getJobsInQueue(queueType: string): Promise> { - return channels.getJobsInQueue(queueType); -} - -function insertJob(job: Readonly): Promise { - return channels.insertJob(job); -} - -function deleteJob(id: string): Promise { - return channels.deleteJob(id); -} - -function processGroupCallRingRequest( - ringId: bigint -): Promise { - return channels.processGroupCallRingRequest(ringId); -} - -function processGroupCallRingCancelation(ringId: bigint): Promise { - return channels.processGroupCallRingCancelation(ringId); -} - -async function cleanExpiredGroupCallRings(): Promise { - await channels.cleanExpiredGroupCallRings(); -} - -async function updateAllConversationColors( - conversationColor?: ConversationColorType, - customColorData?: { - id: string; - value: CustomColorType; - } -): Promise { - return channels.updateAllConversationColors( - conversationColor, - customColorData - ); -} - -async function removeAllProfileKeyCredentials(): Promise { - return channels.removeAllProfileKeyCredentials(); -} - -function getMaxMessageCounter(): Promise { - return channels.getMaxMessageCounter(); -} - -function getStatisticsForLogging(): Promise> { - return channels.getStatisticsForLogging(); -} diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index f423b59854..d2e4b29e1e 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -344,6 +344,17 @@ export type GetConversationRangeCenteredOnMessageResultType = metrics: ConversationMetricsType; }>; +export type MessageAttachmentsCursorType = Readonly<{ + done: boolean; + runId: string; + count: number; +}>; + +export type GetKnownMessageAttachmentsResultType = Readonly<{ + cursor: MessageAttachmentsCursorType; + attachments: ReadonlyArray; +}>; + export type DataInterface = { close: () => Promise; removeDB: () => Promise; @@ -777,17 +788,24 @@ export type ServerInterface = DataInterface & { key: string; }) => Promise; - removeKnownAttachments: ( - allAttachments: Array + getKnownMessageAttachments: ( + cursor?: MessageAttachmentsCursorType + ) => Promise; + finishGetKnownMessageAttachments: ( + cursor: MessageAttachmentsCursorType + ) => Promise; + getKnownConversationAttachments: () => Promise>; + removeKnownStickers: ( + allStickers: ReadonlyArray ) => Promise>; - removeKnownStickers: (allStickers: Array) => Promise>; removeKnownDraftAttachments: ( - allStickers: Array + allStickers: ReadonlyArray ) => Promise>; getAllBadgeImageFileLocalPaths: () => Promise>; }; -export type ClientInterface = DataInterface & { +// Differing signature on client/server +export type ClientExclusiveInterface = { // Differing signature on client/server updateConversation: (data: ConversationType) => void; @@ -870,21 +888,10 @@ export type ClientInterface = DataInterface & { cleanupOrphanedAttachments: () => Promise; ensureFilePermissions: () => Promise; - _jobs: { [id: string]: ClientJobType }; - // To decide whether to use IPC to use the database in the main process or // use the db already running in the renderer. goBackToMainProcess: () => Promise; startInRendererProcess: (isTesting?: boolean) => Promise; }; -export type ClientJobType = { - fnName: string; - start: number; - resolve?: (value: unknown) => void; - reject?: (error: Error) => void; - - // Only in DEBUG mode - complete?: boolean; - args?: ReadonlyArray; -}; +export type ClientInterface = DataInterface & ClientExclusiveInterface; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index c8db90296a..5b928d6630 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -6,6 +6,7 @@ import { join } from 'path'; import mkdirp from 'mkdirp'; import rimraf from 'rimraf'; +import { randomBytes } from 'crypto'; import type { Database, Statement } from 'better-sqlite3'; import SQL from 'better-sqlite3'; import pProps from 'p-props'; @@ -64,7 +65,6 @@ import { getById, bulkAdd, createOrUpdate, - TableIterator, setUserVersion, getUserVersion, getSchemaVersion, @@ -80,6 +80,7 @@ import type { DeleteSentProtoRecipientResultType, EmojiType, GetConversationRangeCenteredOnMessageResultType, + GetKnownMessageAttachmentsResultType, GetUnreadByConversationAndMarkReadResultType, IdentityKeyIdType, StoredIdentityKeyType, @@ -87,6 +88,7 @@ import type { ItemKeyType, StoredItemType, ConversationMessageStatsType, + MessageAttachmentsCursorType, MessageMetricsType, MessageType, MessageTypeUnhydrated, @@ -344,7 +346,9 @@ const dataInterface: ServerInterface = { initialize, initializeRenderer, - removeKnownAttachments, + getKnownMessageAttachments, + finishGetKnownMessageAttachments, + getKnownConversationAttachments, removeKnownStickers, removeKnownDraftAttachments, getAllBadgeImageFileLocalPaths, @@ -924,7 +928,7 @@ async function insertSentProto( ` ); - for (const messageId of messageIds) { + for (const messageId of new Set(messageIds)) { messageStatement.run({ id, messageId, @@ -4512,6 +4516,11 @@ async function _deleteAllStoryDistributions(): Promise { async function createNewStoryDistribution( distribution: StoryDistributionWithMembersType ): Promise { + strictAssert( + distribution.name, + 'Distribution list does not have a valid name' + ); + const db = getInstance(); db.transaction(() => { @@ -4613,6 +4622,18 @@ function modifyStoryDistributionSync( db: Database, payload: StoryDistributionForDatabase ): void { + if (payload.deletedAtTimestamp) { + strictAssert( + !payload.name, + 'Attempt to delete distribution list but still has a name' + ); + } else { + strictAssert( + payload.name, + 'Cannot clear distribution list name without deletedAtTimestamp set' + ); + } + prepare( db, ` @@ -5079,39 +5100,129 @@ function getExternalDraftFilesForConversation( return files; } -async function removeKnownAttachments( - allAttachments: Array -): Promise> { +async function getKnownMessageAttachments( + cursor?: MessageAttachmentsCursorType +): Promise { const db = getInstance(); - const lookup: Dictionary = fromPairs( - map(allAttachments, file => [file, true]) - ); - const chunkSize = 500; + const result = new Set(); + const chunkSize = 1000; - const total = getMessageCountSync(); - logger.info( - `removeKnownAttachments: About to iterate through ${total} messages` - ); + return db.transaction(() => { + let count = cursor?.count ?? 0; - let count = 0; + strictAssert( + !cursor?.done, + 'getKnownMessageAttachments: iteration cannot be restarted' + ); - for (const message of new TableIterator(db, 'messages')) { - const externalFiles = getExternalFilesForMessage(message); - forEach(externalFiles, file => { - delete lookup[file]; - }); - count += 1; + let runId: string; + if (cursor === undefined) { + runId = randomBytes(8).toString('hex'); + + const total = getMessageCountSync(); + logger.info( + `getKnownMessageAttachments(${runId}): ` + + `Starting iteration through ${total} messages` + ); + + db.exec( + ` + CREATE TEMP TABLE tmp_${runId}_updated_messages + (rowid INTEGER PRIMARY KEY ASC); + + INSERT INTO tmp_${runId}_updated_messages (rowid) + SELECT rowid FROM messages; + + CREATE TEMP TRIGGER tmp_${runId}_message_updates + UPDATE OF json ON messages + BEGIN + INSERT OR IGNORE INTO tmp_${runId}_updated_messages (rowid) + VALUES (NEW.rowid); + END; + + CREATE TEMP TRIGGER tmp_${runId}_message_inserts + AFTER INSERT ON messages + BEGIN + INSERT OR IGNORE INTO tmp_${runId}_updated_messages (rowid) + VALUES (NEW.rowid); + END; + ` + ); + } else { + ({ runId } = cursor); + } + + const rowids: Array = db + .prepare( + ` + DELETE FROM tmp_${runId}_updated_messages + RETURNING rowid + LIMIT $chunkSize; + ` + ) + .pluck() + .all({ chunkSize }); + + const messages = batchMultiVarQuery( + db, + rowids, + (batch: Array): Array => { + const query = db.prepare( + `SELECT json FROM messages WHERE rowid IN (${Array(batch.length) + .fill('?') + .join(',')});` + ); + const rows: JSONRows = query.all(batch); + return rows.map(row => jsonToObject(row.json)); + } + ); + + for (const message of messages) { + const externalFiles = getExternalFilesForMessage(message); + forEach(externalFiles, file => result.add(file)); + count += 1; + } + + const done = messages.length < chunkSize; + return { + attachments: Array.from(result), + cursor: { runId, count, done }, + }; + })(); +} + +async function finishGetKnownMessageAttachments({ + runId, + count, + done, +}: MessageAttachmentsCursorType): Promise { + const db = getInstance(); + + const logId = `finishGetKnownMessageAttachments(${runId})`; + if (!done) { + logger.warn(`${logId}: iteration not finished`); } - logger.info(`removeKnownAttachments: Done processing ${count} messages`); + logger.info(`${logId}: reached the end after processing ${count} messages`); + db.exec(` + DROP TABLE tmp_${runId}_updated_messages; + DROP TRIGGER tmp_${runId}_message_updates; + DROP TRIGGER tmp_${runId}_message_inserts; + `); +} + +async function getKnownConversationAttachments(): Promise> { + const db = getInstance(); + const result = new Set(); + const chunkSize = 500; let complete = false; - count = 0; let id = ''; const conversationTotal = await getConversationCount(); logger.info( - `removeKnownAttachments: About to iterate through ${conversationTotal} conversations` + 'getKnownConversationAttachments: About to iterate through ' + + `${conversationTotal}` ); const fetchConversations = db.prepare( @@ -5134,9 +5245,7 @@ async function removeKnownAttachments( ); conversations.forEach(conversation => { const externalFiles = getExternalFilesForConversation(conversation); - externalFiles.forEach(file => { - delete lookup[file]; - }); + externalFiles.forEach(file => result.add(file)); }); const lastMessage: ConversationType | undefined = last(conversations); @@ -5144,16 +5253,15 @@ async function removeKnownAttachments( ({ id } = lastMessage); } complete = conversations.length < chunkSize; - count += conversations.length; } - logger.info(`removeKnownAttachments: Done processing ${count} conversations`); + logger.info('getKnownConversationAttachments: Done processing'); - return Object.keys(lookup); + return Array.from(result); } async function removeKnownStickers( - allStickers: Array + allStickers: ReadonlyArray ): Promise> { const db = getInstance(); const lookup: Dictionary = fromPairs( @@ -5204,7 +5312,7 @@ async function removeKnownStickers( } async function removeKnownDraftAttachments( - allStickers: Array + allStickers: ReadonlyArray ): Promise> { const db = getInstance(); const lookup: Dictionary = fromPairs( diff --git a/ts/sql/main.ts b/ts/sql/main.ts index 6af8d312ff..994d0b54fa 100644 --- a/ts/sql/main.ts +++ b/ts/sql/main.ts @@ -10,6 +10,7 @@ import { strictAssert } from '../util/assert'; import { explodePromise } from '../util/explodePromise'; import type { LoggerType } from '../types/Logging'; import { isCorruptionError } from './errors'; +import type DB from './Server'; const MIN_TRACE_DURATION = 40; @@ -32,9 +33,8 @@ export type WorkerRequest = Readonly< } | { type: 'sqlCall'; - method: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - args: ReadonlyArray; + method: keyof typeof DB; + args: ReadonlyArray; } >; @@ -164,8 +164,10 @@ export class MainSQL { await this.send({ type: 'removeDB' }); } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - public async sqlCall(method: string, args: ReadonlyArray): Promise { + public async sqlCall( + method: Method, + ...args: Parameters + ): Promise> { if (this.onReady) { await this.onReady; } @@ -175,8 +177,7 @@ export class MainSQL { } type SqlCallResult = Readonly<{ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - result: any; + result: ReturnType; duration: number; }>;