// Copyright 2021 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only import type { Database } from '@signalapp/better-sqlite3'; import { isNumber, last } from 'lodash'; export type EmptyQuery = []; export type ArrayQuery = Array>; export type Query = { [key: string]: null | number | bigint | string | Uint8Array; }; export type JSONRows = Array<{ readonly json: string }>; export type TableType = | 'attachment_downloads' | 'conversations' | 'identityKeys' | 'items' | 'kyberPreKeys' | 'messages' | 'preKeys' | 'senderKeys' | 'sessions' | 'signedPreKeys' | 'stickers' | 'unprocessed'; // This value needs to be below SQLITE_MAX_VARIABLE_NUMBER. const MAX_VARIABLE_COUNT = 100; export function objectToJSON(data: T): string { return JSON.stringify(data); } export function jsonToObject(json: string): T { return JSON.parse(json); } export type QueryTemplateParam = string | number | null | undefined; export type QueryFragmentValue = QueryFragment | QueryTemplateParam; export type QueryFragment = [ { fragment: string }, ReadonlyArray ]; /** * You can use tagged template literals to build "fragments" of SQL queries * * ```ts * const [query, params] = sql` * SELECT * FROM examples * WHERE groupId = ${groupId} * ORDER BY timestamp ${asc ? sqlFragment`ASC` : sqlFragment`DESC`} * `; * ``` * * SQL Fragments can contain other SQL fragments, but must be finalized with * `sql` before being passed to `Database#prepare`. * * The name `sqlFragment` comes from several editors that support SQL syntax * highlighting inside JavaScript template literals. */ export function sqlFragment( strings: TemplateStringsArray, ...values: ReadonlyArray ): QueryFragment { let query = ''; const params: Array = []; strings.forEach((string, index) => { const value = values[index]; query += string; if (index < values.length) { if (Array.isArray(value)) { const [{ fragment }, fragmentParams] = value; query += fragment; params.push(...fragmentParams); } else { query += '?'; params.push(value); } } }); return [{ fragment: query }, params]; } export function sqlConstant(value: QueryTemplateParam): QueryFragment { let fragment; if (value == null) { fragment = 'NULL'; } else if (typeof value === 'number') { fragment = `${value}`; } else if (typeof value === 'boolean') { fragment = `${value}`; } else { fragment = `'${value}'`; } return [{ fragment }, []]; } /** * Like `Array.prototype.join`, but for SQL fragments. */ export function sqlJoin( items: ReadonlyArray, separator: string ): QueryFragment { let query = ''; const params: Array = []; items.forEach((item, index) => { const [{ fragment }, fragmentParams] = sqlFragment`${item}`; query += fragment; params.push(...fragmentParams); if (index < items.length - 1) { query += separator; } }); return [{ fragment: query }, params]; } export type QueryTemplate = [string, ReadonlyArray]; /** * You can use tagged template literals to build SQL queries * that can be passed to `Database#prepare`. * * ```ts * const [query, params] = sql` * SELECT * FROM examples * WHERE groupId = ${groupId} * ORDER BY timestamp ASC * `; * db.prepare(query).all(params); * ``` * * SQL queries can contain other SQL fragments, but cannot contain other SQL * queries. * * The name `sql` comes from several editors that support SQL syntax * highlighting inside JavaScript template literals. */ export function sql( strings: TemplateStringsArray, ...values: ReadonlyArray ): QueryTemplate { const [{ fragment }, params] = sqlFragment(strings, ...values); return [fragment, params]; } type QueryPlanRow = Readonly<{ id: number; parent: number; details: string; }>; type QueryPlan = Readonly<{ query: string; plan: ReadonlyArray; }>; /** * Returns typed objects of the query plan for the given query. * * * ```ts * const [query, params] = sql` * SELECT * FROM examples * WHERE groupId = ${groupId} * ORDER BY timestamp ASC * `; * log.info('Query plan', explainQueryPlan(db, [query, params])); * db.prepare(query).all(params); * ``` */ export function explainQueryPlan( db: Database, template: QueryTemplate ): QueryPlan { const [query, params] = template; const plan = db.prepare(`EXPLAIN QUERY PLAN ${query}`).all(params); return { query, plan }; } // // Database helpers // export function getSQLiteVersion(db: Database): string { const { sqlite_version: version } = db .prepare('select sqlite_version() AS sqlite_version') .get(); return version; } export function getSchemaVersion(db: Database): number { return db.pragma('schema_version', { simple: true }); } export function setUserVersion(db: Database, version: number): void { if (!isNumber(version)) { throw new Error(`setUserVersion: version ${version} is not a number`); } db.pragma(`user_version = ${version}`); } export function getUserVersion(db: Database): number { return db.pragma('user_version', { simple: true }); } export function getSQLCipherVersion(db: Database): string | undefined { return db.pragma('cipher_version', { simple: true }); } // // Various table helpers // export function batchMultiVarQuery( db: Database, values: ReadonlyArray, query: (batch: ReadonlyArray) => void ): []; export function batchMultiVarQuery( db: Database, values: ReadonlyArray, query: (batch: ReadonlyArray) => Array ): Array; export function batchMultiVarQuery( db: Database, values: ReadonlyArray, query: | ((batch: ReadonlyArray) => void) | ((batch: ReadonlyArray) => Array) ): Array { if (values.length > MAX_VARIABLE_COUNT) { const result: Array = []; db.transaction(() => { for (let i = 0; i < values.length; i += MAX_VARIABLE_COUNT) { const batch = values.slice(i, i + MAX_VARIABLE_COUNT); const batchResult = query(batch); if (Array.isArray(batchResult)) { result.push(...batchResult); } } })(); return result; } const result = query(values); return Array.isArray(result) ? result : []; } export function createOrUpdate( db: Database, table: TableType, data: Record & { id: Key } ): void { const { id } = data; if (!id) { throw new Error('createOrUpdate: Provided data did not have a truthy id'); } db.prepare( ` INSERT OR REPLACE INTO ${table} ( id, json ) values ( $id, $json ) ` ).run({ id, json: objectToJSON(data), }); } export function bulkAdd( db: Database, table: TableType, array: Array & { id: string | number }> ): void { db.transaction(() => { for (const data of array) { createOrUpdate(db, table, data); } })(); } export function getById( db: Database, table: TableType, id: Key ): Result | undefined { const row = db .prepare( ` SELECT * FROM ${table} WHERE id = $id; ` ) .get({ id, }); if (!row) { return undefined; } return jsonToObject(row.json); } export function removeById( db: Database, tableName: TableType, id: Key | Array ): number { const table = sqlConstant(tableName); if (!Array.isArray(id)) { const [query, params] = sql` DELETE FROM ${table} WHERE id = ${id}; `; return db.prepare(query).run(params).changes; } if (!id.length) { throw new Error('removeById: No ids to delete!'); } let totalChanges = 0; const removeByIdsSync = (ids: ReadonlyArray): void => { const [query, params] = sql` DELETE FROM ${table} WHERE id IN (${sqlJoin(ids, ', ')}); `; totalChanges += db.prepare(query).run(params).changes; }; batchMultiVarQuery(db, id, removeByIdsSync); return totalChanges; } export function removeAllFromTable(db: Database, table: TableType): number { return db.prepare(`DELETE FROM ${table};`).run().changes; } export function getAllFromTable(db: Database, table: TableType): Array { const rows: JSONRows = db .prepare(`SELECT json FROM ${table};`) .all(); return rows.map(row => jsonToObject(row.json)); } export function getCountFromTable(db: Database, table: TableType): number { const result: null | number = db .prepare(`SELECT count(*) from ${table};`) .pluck(true) .get(); if (isNumber(result)) { return result; } throw new Error(`getCountFromTable: Unable to get count from table ${table}`); } export class TableIterator { constructor( private readonly db: Database, private readonly table: TableType, private readonly pageSize = 500 ) {} *[Symbol.iterator](): Iterator { const fetchObject = this.db.prepare( ` SELECT json FROM ${this.table} WHERE id > $id ORDER BY id ASC LIMIT $pageSize; ` ); let complete = false; let id = ''; while (!complete) { const rows: JSONRows = fetchObject.all({ id, pageSize: this.pageSize, }); const messages: Array = rows.map(row => jsonToObject(row.json) ); yield* messages; const lastMessage: ObjectType | undefined = last(messages); if (lastMessage) { ({ id } = lastMessage); } complete = messages.length < this.pageSize; } } }