Graceful handling of readonly DB error

This commit is contained in:
Fedor Indutny 2023-03-02 09:59:18 -08:00 committed by GitHub
parent 1e7d658109
commit bd40a7fb98
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 98 additions and 27 deletions

View file

@ -1573,6 +1573,20 @@ const runSQLCorruptionHandler = async () => {
await onDatabaseError(Errors.toLogFormat(error));
};
const runSQLReadonlyHandler = async () => {
// This is a glorified event handler. Normally, this promise never resolves,
// but if there is a corruption error triggered by any query that we run
// against the database - the promise will resolve and we will call
// `onDatabaseError`.
const error = await sql.whenReadonly();
getLogger().error(
`Detected readonly sql database in main process: ${error.message}`
);
throw error;
};
async function initializeSQL(
userDataPath: string
): Promise<{ ok: true; error: undefined } | { ok: false; error: Error }> {
@ -1619,6 +1633,7 @@ async function initializeSQL(
// Only if we've initialized things successfully do we set up the corruption handler
drop(runSQLCorruptionHandler());
drop(runSQLReadonlyHandler());
return { ok: true, error: undefined };
}
@ -1669,6 +1684,11 @@ ipc.on('database-error', (_event: Electron.Event, error: string) => {
drop(onDatabaseError(error));
});
ipc.on('database-readonly', (_event: Electron.Event, error: string) => {
// Just let global_errors.ts handle it
throw new Error(error);
});
function loadPreferredSystemLocales(): Array<string> {
return getEnvironment() === Environment.Test
? ['en']

View file

@ -65,7 +65,7 @@ import type {
StoredSignedPreKeyType,
} from './Interface';
import Server from './Server';
import { isCorruptionError } from './errors';
import { parseSqliteError, SqliteErrorKind } from './errors';
import { MINUTE } from '../util/durations';
import { getMessageIdForLogging } from '../util/idForLogging';
@ -309,13 +309,18 @@ function makeChannel(fnName: string) {
// Ignoring this error TS2556: Expected 3 arguments, but got 0 or more.
return await serverFn(...args);
} catch (error) {
if (isCorruptionError(error)) {
const sqliteErrorKind = parseSqliteError(error);
if (sqliteErrorKind === SqliteErrorKind.Corrupted) {
log.error(
'Detected sql corruption in renderer process. ' +
`Restarting the application immediately. Error: ${error.message}`
);
ipc?.send('database-error', error.stack);
} else if (sqliteErrorKind === SqliteErrorKind.Readonly) {
log.error(`Detected readonly sql database: ${error.message}`);
ipc?.send('database-readonly');
}
log.error(
`Renderer SQL channel job (${fnName}) error ${error.message}`
);

View file

@ -1,11 +1,32 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
export function isCorruptionError(error?: Error): boolean {
return (
error?.message?.includes('SQLITE_CORRUPT') ||
error?.message?.includes('database disk image is malformed') ||
error?.message?.includes('file is not a database') ||
false
);
export enum SqliteErrorKind {
Corrupted,
Readonly,
Unknown,
}
export function parseSqliteError(error?: Error): SqliteErrorKind {
const message = error?.message;
if (!message) {
return SqliteErrorKind.Unknown;
}
if (
message.includes('SQLITE_CORRUPT') ||
message.includes('database disk image is malformed') ||
message.includes('file is not a database')
) {
return SqliteErrorKind.Corrupted;
}
if (
message.includes('SQLITE_READONLY') ||
message.includes('attempt to write a readonly database')
) {
return SqliteErrorKind.Readonly;
}
return SqliteErrorKind.Unknown;
}

View file

@ -9,7 +9,7 @@ import { app } from 'electron';
import { strictAssert } from '../util/assert';
import { explodePromise } from '../util/explodePromise';
import type { LoggerType } from '../types/Logging';
import { isCorruptionError } from './errors';
import { parseSqliteError, SqliteErrorKind } from './errors';
import type DB from './Server';
const MIN_TRACE_DURATION = 40;
@ -64,6 +64,11 @@ type PromisePair<T> = {
reject: (error: Error) => void;
};
type KnownErrorResolverType = Readonly<{
kind: SqliteErrorKind;
resolve: (err: Error) => void;
}>;
export class MainSQL {
private readonly worker: Worker;
@ -73,9 +78,8 @@ export class MainSQL {
private readonly onExit: Promise<void>;
// This promise is resolved when any of the queries that we run against the
// database reject with a corruption error (see `isCorruptionError`)
private readonly onCorruption: Promise<Error>;
// Promise resolve callbacks for corruption and readonly errors.
private errorResolvers = new Array<KnownErrorResolverType>();
private seq = 0;
@ -88,10 +92,6 @@ export class MainSQL {
const scriptDir = join(app.getAppPath(), 'ts', 'sql', 'mainWorker.js');
this.worker = new Worker(scriptDir);
const { promise: onCorruption, resolve: resolveCorruption } =
explodePromise<Error>();
this.onCorruption = onCorruption;
this.worker.on('message', (wrappedResponse: WrappedWorkerResponse) => {
if (wrappedResponse.type === 'log') {
const { level, args } = wrappedResponse;
@ -110,9 +110,7 @@ export class MainSQL {
if (error) {
const errorObj = new Error(error);
if (isCorruptionError(errorObj)) {
resolveCorruption(errorObj);
}
this.onError(errorObj);
pair.reject(errorObj);
} else {
@ -120,9 +118,9 @@ export class MainSQL {
}
});
this.onExit = new Promise<void>(resolve => {
this.worker.once('exit', resolve);
});
const { promise: onExit, resolve: resolveOnExit } = explodePromise<void>();
this.onExit = onExit;
this.worker.once('exit', resolveOnExit);
}
public async initialize({
@ -148,7 +146,15 @@ export class MainSQL {
}
public whenCorrupted(): Promise<Error> {
return this.onCorruption;
const { promise, resolve } = explodePromise<Error>();
this.errorResolvers.push({ kind: SqliteErrorKind.Corrupted, resolve });
return promise;
}
public whenReadonly(): Promise<Error> {
const { promise, resolve } = explodePromise<Error>();
this.errorResolvers.push({ kind: SqliteErrorKind.Readonly, resolve });
return promise;
}
public async close(): Promise<void> {
@ -199,9 +205,8 @@ export class MainSQL {
const { seq } = this;
this.seq += 1;
const result = new Promise<Response>((resolve, reject) => {
this.onResponse.set(seq, { resolve, reject });
});
const { promise: result, resolve, reject } = explodePromise<Response>();
this.onResponse.set(seq, { resolve, reject });
const wrappedRequest: WrappedWorkerRequest = {
seq,
@ -211,4 +216,24 @@ export class MainSQL {
return result;
}
private onError(error: Error): void {
const errorKind = parseSqliteError(error);
if (errorKind === SqliteErrorKind.Unknown) {
return;
}
const resolvers = new Array<(error: Error) => void>();
this.errorResolvers = this.errorResolvers.filter(entry => {
if (entry.kind === errorKind) {
resolvers.push(entry.resolve);
return false;
}
return true;
});
for (const resolve of resolvers) {
resolve(error);
}
}
}