Parallelize SQL queries
This commit is contained in:
Fedor Indutny
GitHub
parent
86b4da1ec2
commit
c64762858e
178 changed files with 3377 additions and 3618 deletions
406
ts/sql/main.ts
406
ts/sql/main.ts
|
|
@ -10,10 +10,15 @@ import { strictAssert } from '../util/assert';
|
|||
import { explodePromise } from '../util/explodePromise';
|
||||
import type { LoggerType } from '../types/Logging';
|
||||
import { SqliteErrorKind } from './errors';
|
||||
import type DB from './Server';
|
||||
import type {
|
||||
ServerReadableDirectInterface,
|
||||
ServerWritableDirectInterface,
|
||||
} from './Interface';
|
||||
|
||||
const MIN_TRACE_DURATION = 40;
|
||||
|
||||
const WORKER_COUNT = 4;
|
||||
|
||||
export type InitializeOptions = Readonly<{
|
||||
appVersion: string;
|
||||
configDir: string;
|
||||
|
|
@ -25,16 +30,19 @@ export type WorkerRequest = Readonly<
|
|||
| {
|
||||
type: 'init';
|
||||
options: Omit<InitializeOptions, 'logger'>;
|
||||
isPrimary: boolean;
|
||||
}
|
||||
| {
|
||||
type: 'close';
|
||||
type: 'close' | 'removeDB';
|
||||
}
|
||||
| {
|
||||
type: 'removeDB';
|
||||
type: 'sqlCall:read';
|
||||
method: keyof ServerReadableDirectInterface;
|
||||
args: ReadonlyArray<unknown>;
|
||||
}
|
||||
| {
|
||||
type: 'sqlCall';
|
||||
method: keyof typeof DB;
|
||||
type: 'sqlCall:write';
|
||||
method: keyof ServerWritableDirectInterface;
|
||||
args: ReadonlyArray<unknown>;
|
||||
}
|
||||
>;
|
||||
|
|
@ -71,14 +79,26 @@ type KnownErrorResolverType = Readonly<{
|
|||
resolve: (err: Error) => void;
|
||||
}>;
|
||||
|
||||
type CreateWorkerResultType = Readonly<{
|
||||
worker: Worker;
|
||||
onExit: Promise<void>;
|
||||
}>;
|
||||
|
||||
type PoolEntry = {
|
||||
readonly worker: Worker;
|
||||
load: number;
|
||||
};
|
||||
|
||||
export class MainSQL {
|
||||
private readonly worker: Worker;
|
||||
private readonly pool = new Array<PoolEntry>();
|
||||
|
||||
private pauseWaiters: Array<() => void> | undefined;
|
||||
|
||||
private isReady = false;
|
||||
|
||||
private onReady: Promise<void> | undefined;
|
||||
|
||||
private readonly onExit: Promise<void>;
|
||||
private readonly onExit: Promise<unknown>;
|
||||
|
||||
// Promise resolve callbacks for corruption and readonly errors.
|
||||
private errorResolvers = new Array<KnownErrorResolverType>();
|
||||
|
|
@ -93,10 +113,246 @@ export class MainSQL {
|
|||
private shouldTimeQueries = false;
|
||||
|
||||
constructor() {
|
||||
const scriptDir = join(app.getAppPath(), 'ts', 'sql', 'mainWorker.js');
|
||||
this.worker = new Worker(scriptDir);
|
||||
const exitPromises = new Array<Promise<void>>();
|
||||
for (let i = 0; i < WORKER_COUNT; i += 1) {
|
||||
const { worker, onExit } = this.createWorker();
|
||||
this.pool.push({ worker, load: 0 });
|
||||
|
||||
this.worker.on('message', (wrappedResponse: WrappedWorkerResponse) => {
|
||||
exitPromises.push(onExit);
|
||||
}
|
||||
this.onExit = Promise.all(exitPromises);
|
||||
}
|
||||
|
||||
public async initialize({
|
||||
appVersion,
|
||||
configDir,
|
||||
key,
|
||||
logger,
|
||||
}: InitializeOptions): Promise<void> {
|
||||
if (this.isReady || this.onReady) {
|
||||
throw new Error('Already initialized');
|
||||
}
|
||||
|
||||
this.shouldTimeQueries = Boolean(process.env.TIME_QUERIES);
|
||||
|
||||
this.logger = logger;
|
||||
|
||||
this.onReady = (async () => {
|
||||
const primary = this.pool[0];
|
||||
const rest = this.pool.slice(1);
|
||||
|
||||
await this.send(primary, {
|
||||
type: 'init',
|
||||
options: { appVersion, configDir, key },
|
||||
isPrimary: true,
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
rest.map(worker =>
|
||||
this.send(worker, {
|
||||
type: 'init',
|
||||
options: { appVersion, configDir, key },
|
||||
isPrimary: false,
|
||||
})
|
||||
)
|
||||
);
|
||||
})();
|
||||
|
||||
await this.onReady;
|
||||
|
||||
this.onReady = undefined;
|
||||
this.isReady = true;
|
||||
}
|
||||
|
||||
public pauseWriteAccess(): void {
|
||||
strictAssert(this.pauseWaiters == null, 'Already paused');
|
||||
|
||||
this.pauseWaiters = [];
|
||||
}
|
||||
|
||||
public resumeWriteAccess(): void {
|
||||
const { pauseWaiters } = this;
|
||||
strictAssert(pauseWaiters != null, 'Not paused');
|
||||
this.pauseWaiters = undefined;
|
||||
|
||||
for (const waiter of pauseWaiters) {
|
||||
waiter();
|
||||
}
|
||||
}
|
||||
|
||||
public whenCorrupted(): Promise<Error> {
|
||||
const { promise, resolve } = explodePromise<Error>();
|
||||
this.errorResolvers.push({ kind: SqliteErrorKind.Corrupted, resolve });
|
||||
return promise;
|
||||
}
|
||||
|
||||
public whenReadonly(): Promise<Error> {
|
||||
const { promise, resolve } = explodePromise<Error>();
|
||||
this.errorResolvers.push({ kind: SqliteErrorKind.Readonly, resolve });
|
||||
return promise;
|
||||
}
|
||||
|
||||
public async close(): Promise<void> {
|
||||
if (!this.isReady) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
await this.terminate({ type: 'close' });
|
||||
await this.onExit;
|
||||
}
|
||||
|
||||
public async removeDB(): Promise<void> {
|
||||
await this.terminate({ type: 'removeDB' });
|
||||
}
|
||||
|
||||
public async sqlRead<Method extends keyof ServerReadableDirectInterface>(
|
||||
method: Method,
|
||||
...args: Parameters<ServerReadableDirectInterface[Method]>
|
||||
): Promise<ReturnType<ServerReadableDirectInterface[Method]>> {
|
||||
type SqlCallResult = Readonly<{
|
||||
result: ReturnType<ServerReadableDirectInterface[Method]>;
|
||||
duration: number;
|
||||
}>;
|
||||
|
||||
// pageMessages runs over several queries and needs to have access to
|
||||
// the same temporary table.
|
||||
const isPaging = method === 'pageMessages';
|
||||
|
||||
const entry = isPaging ? this.pool.at(-1) : this.getWorker();
|
||||
strictAssert(entry != null, 'Must have a pool entry');
|
||||
|
||||
const { result, duration } = await this.send<SqlCallResult>(entry, {
|
||||
type: 'sqlCall:read',
|
||||
method,
|
||||
args,
|
||||
});
|
||||
|
||||
this.traceDuration(method, duration);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async sqlWrite<Method extends keyof ServerWritableDirectInterface>(
|
||||
method: Method,
|
||||
...args: Parameters<ServerWritableDirectInterface[Method]>
|
||||
): Promise<ReturnType<ServerWritableDirectInterface[Method]>> {
|
||||
type Result = ReturnType<ServerWritableDirectInterface[Method]>;
|
||||
type SqlCallResult = Readonly<{
|
||||
result: Result;
|
||||
duration: number;
|
||||
}>;
|
||||
|
||||
while (this.pauseWaiters != null) {
|
||||
const { promise, resolve } = explodePromise<void>();
|
||||
this.pauseWaiters.push(resolve);
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await promise;
|
||||
}
|
||||
|
||||
// Special case since we need to broadcast this to every pool entry.
|
||||
if (method === 'removeDB') {
|
||||
return (await this.removeDB()) as Result;
|
||||
}
|
||||
|
||||
const primary = this.pool[0];
|
||||
|
||||
const { result, duration } = await this.send<SqlCallResult>(primary, {
|
||||
type: 'sqlCall:write',
|
||||
method,
|
||||
args,
|
||||
});
|
||||
|
||||
this.traceDuration(method, duration);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async send<Response>(
|
||||
entry: PoolEntry,
|
||||
request: WorkerRequest
|
||||
): Promise<Response> {
|
||||
if (request.type === 'sqlCall:read' || request.type === 'sqlCall:write') {
|
||||
if (this.onReady) {
|
||||
await this.onReady;
|
||||
}
|
||||
|
||||
if (!this.isReady) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
}
|
||||
|
||||
const { seq } = this;
|
||||
// eslint-disable-next-line no-bitwise
|
||||
this.seq = (this.seq + 1) >>> 0;
|
||||
|
||||
const { promise: result, resolve, reject } = explodePromise<Response>();
|
||||
this.onResponse.set(seq, { resolve, reject });
|
||||
|
||||
const wrappedRequest: WrappedWorkerRequest = {
|
||||
seq,
|
||||
request,
|
||||
};
|
||||
entry.worker.postMessage(wrappedRequest);
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
entry.load += 1;
|
||||
return await result;
|
||||
} finally {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
entry.load -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
private async terminate(request: WorkerRequest): Promise<void> {
|
||||
const primary = this.pool[0];
|
||||
const rest = this.pool.slice(1);
|
||||
|
||||
// Terminate non-primary workers first
|
||||
await Promise.all(rest.map(worker => this.send(worker, request)));
|
||||
|
||||
// Primary last
|
||||
await this.send(primary, request);
|
||||
}
|
||||
|
||||
private onError(errorKind: SqliteErrorKind, error: Error): void {
|
||||
if (errorKind === SqliteErrorKind.Unknown) {
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvers = new Array<(error: Error) => void>();
|
||||
this.errorResolvers = this.errorResolvers.filter(entry => {
|
||||
if (entry.kind === errorKind) {
|
||||
resolvers.push(entry.resolve);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
for (const resolve of resolvers) {
|
||||
resolve(error);
|
||||
}
|
||||
}
|
||||
|
||||
private traceDuration(method: string, duration: number): void {
|
||||
if (this.shouldTimeQueries && !app.isPackaged) {
|
||||
const twoDecimals = Math.round(100 * duration) / 100;
|
||||
this.logger?.info(`MainSQL query: ${method}, duration=${twoDecimals}ms`);
|
||||
}
|
||||
if (duration > MIN_TRACE_DURATION) {
|
||||
strictAssert(this.logger !== undefined, 'Logger not initialized');
|
||||
this.logger.info(
|
||||
`MainSQL: slow query ${method} duration=${Math.round(duration)}ms`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private createWorker(): CreateWorkerResultType {
|
||||
const scriptPath = join(app.getAppPath(), 'ts', 'sql', 'mainWorker.js');
|
||||
|
||||
const worker = new Worker(scriptPath);
|
||||
|
||||
worker.on('message', (wrappedResponse: WrappedWorkerResponse) => {
|
||||
if (wrappedResponse.type === 'log') {
|
||||
const { level, args } = wrappedResponse;
|
||||
strictAssert(this.logger !== undefined, 'Logger not initialized');
|
||||
|
|
@ -123,129 +379,21 @@ export class MainSQL {
|
|||
});
|
||||
|
||||
const { promise: onExit, resolve: resolveOnExit } = explodePromise<void>();
|
||||
this.onExit = onExit;
|
||||
this.worker.once('exit', resolveOnExit);
|
||||
worker.once('exit', resolveOnExit);
|
||||
|
||||
return { worker, onExit };
|
||||
}
|
||||
|
||||
public async initialize({
|
||||
appVersion,
|
||||
configDir,
|
||||
key,
|
||||
logger,
|
||||
}: InitializeOptions): Promise<void> {
|
||||
if (this.isReady || this.onReady) {
|
||||
throw new Error('Already initialized');
|
||||
}
|
||||
|
||||
this.shouldTimeQueries = Boolean(process.env.TIME_QUERIES);
|
||||
|
||||
this.logger = logger;
|
||||
|
||||
this.onReady = this.send({
|
||||
type: 'init',
|
||||
options: { appVersion, configDir, key },
|
||||
});
|
||||
|
||||
await this.onReady;
|
||||
|
||||
this.onReady = undefined;
|
||||
this.isReady = true;
|
||||
}
|
||||
|
||||
public whenCorrupted(): Promise<Error> {
|
||||
const { promise, resolve } = explodePromise<Error>();
|
||||
this.errorResolvers.push({ kind: SqliteErrorKind.Corrupted, resolve });
|
||||
return promise;
|
||||
}
|
||||
|
||||
public whenReadonly(): Promise<Error> {
|
||||
const { promise, resolve } = explodePromise<Error>();
|
||||
this.errorResolvers.push({ kind: SqliteErrorKind.Readonly, resolve });
|
||||
return promise;
|
||||
}
|
||||
|
||||
public async close(): Promise<void> {
|
||||
if (!this.isReady) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
await this.send({ type: 'close' });
|
||||
await this.onExit;
|
||||
}
|
||||
|
||||
public async removeDB(): Promise<void> {
|
||||
await this.send({ type: 'removeDB' });
|
||||
}
|
||||
|
||||
public async sqlCall<Method extends keyof typeof DB>(
|
||||
method: Method,
|
||||
...args: Parameters<typeof DB[Method]>
|
||||
): Promise<ReturnType<typeof DB[Method]>> {
|
||||
if (this.onReady) {
|
||||
await this.onReady;
|
||||
}
|
||||
|
||||
if (!this.isReady) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
type SqlCallResult = Readonly<{
|
||||
result: ReturnType<typeof DB[Method]>;
|
||||
duration: number;
|
||||
}>;
|
||||
|
||||
const { result, duration } = await this.send<SqlCallResult>({
|
||||
type: 'sqlCall',
|
||||
method,
|
||||
args,
|
||||
});
|
||||
|
||||
if (this.shouldTimeQueries && !app.isPackaged) {
|
||||
const twoDecimals = Math.round(100 * duration) / 100;
|
||||
this.logger?.info(`MainSQL query: ${method}, duration=${twoDecimals}ms`);
|
||||
}
|
||||
if (duration > MIN_TRACE_DURATION) {
|
||||
strictAssert(this.logger !== undefined, 'Logger not initialized');
|
||||
this.logger.info(
|
||||
`MainSQL: slow query ${method} duration=${Math.round(duration)}ms`
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private async send<Response>(request: WorkerRequest): Promise<Response> {
|
||||
const { seq } = this;
|
||||
this.seq += 1;
|
||||
|
||||
const { promise: result, resolve, reject } = explodePromise<Response>();
|
||||
this.onResponse.set(seq, { resolve, reject });
|
||||
|
||||
const wrappedRequest: WrappedWorkerRequest = {
|
||||
seq,
|
||||
request,
|
||||
};
|
||||
this.worker.postMessage(wrappedRequest);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private onError(errorKind: SqliteErrorKind, error: Error): void {
|
||||
if (errorKind === SqliteErrorKind.Unknown) {
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvers = new Array<(error: Error) => void>();
|
||||
this.errorResolvers = this.errorResolvers.filter(entry => {
|
||||
if (entry.kind === errorKind) {
|
||||
resolvers.push(entry.resolve);
|
||||
return false;
|
||||
// Find first pool entry with minimal load
|
||||
private getWorker(): PoolEntry {
|
||||
let min = this.pool[0];
|
||||
for (const entry of this.pool) {
|
||||
if (min && min.load < entry.load) {
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
for (const resolve of resolvers) {
|
||||
resolve(error);
|
||||
min = entry;
|
||||
}
|
||||
return min;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue