mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-25 06:18:45 +08:00
feat(nbstore): improve nbstore (#9512)
This commit is contained in:
EYHN
@@ -1,54 +1,58 @@
|
||||
import type { OpClient } from '@toeverything/infra/op';
|
||||
|
||||
import { DummyConnection } from '../connection';
|
||||
import { DocFrontend } from '../frontend/doc';
|
||||
import { AwarenessFrontend, BlobFrontend, DocFrontend } from '../frontend';
|
||||
import {
|
||||
type AwarenessRecord,
|
||||
type AwarenessStorage,
|
||||
type BlobRecord,
|
||||
type BlobStorage,
|
||||
type DocRecord,
|
||||
type DocStorage,
|
||||
type DocUpdate,
|
||||
type ListedBlobRecord,
|
||||
type StorageOptions,
|
||||
universalId,
|
||||
} from '../storage';
|
||||
import type { AwarenessSync } from '../sync/awareness';
|
||||
import type { BlobSync } from '../sync/blob';
|
||||
import type { DocSync } from '../sync/doc';
|
||||
import type { WorkerOps } from './ops';
|
||||
import type { WorkerInitOptions, WorkerOps } from './ops';
|
||||
|
||||
export type { WorkerInitOptions } from './ops';
|
||||
|
||||
export class WorkerClient {
|
||||
constructor(
|
||||
private readonly client: OpClient<WorkerOps>,
|
||||
private readonly options: StorageOptions
|
||||
) {}
|
||||
options: WorkerInitOptions
|
||||
) {
|
||||
client.listen();
|
||||
this.client.call('worker.init', options).catch(err => {
|
||||
console.error('error initializing worker', err);
|
||||
});
|
||||
this.docStorage = new WorkerDocStorage(this.client);
|
||||
this.blobStorage = new WorkerBlobStorage(this.client);
|
||||
this.docSync = new WorkerDocSync(this.client);
|
||||
this.blobSync = new WorkerBlobSync(this.client);
|
||||
this.awarenessSync = new WorkerAwarenessSync(this.client);
|
||||
this.docFrontend = new DocFrontend(this.docStorage, this.docSync);
|
||||
this.blobFrontend = new BlobFrontend(this.blobStorage, this.blobSync);
|
||||
this.awarenessFrontend = new AwarenessFrontend(this.awarenessSync);
|
||||
}
|
||||
|
||||
readonly docStorage = new WorkerDocStorage(this.client, this.options);
|
||||
readonly blobStorage = new WorkerBlobStorage(this.client, this.options);
|
||||
readonly awarenessStorage = new WorkerAwarenessStorage(
|
||||
this.client,
|
||||
this.options
|
||||
);
|
||||
readonly docSync = new WorkerDocSync(this.client);
|
||||
readonly blobSync = new WorkerBlobSync(this.client);
|
||||
readonly awarenessSync = new WorkerAwarenessSync(this.client);
|
||||
private readonly docStorage: WorkerDocStorage;
|
||||
private readonly blobStorage: WorkerBlobStorage;
|
||||
private readonly docSync: WorkerDocSync;
|
||||
private readonly blobSync: WorkerBlobSync;
|
||||
private readonly awarenessSync: WorkerAwarenessSync;
|
||||
|
||||
readonly docFrontend = new DocFrontend(this.docStorage, this.docSync);
|
||||
readonly docFrontend: DocFrontend;
|
||||
readonly blobFrontend: BlobFrontend;
|
||||
readonly awarenessFrontend: AwarenessFrontend;
|
||||
}
|
||||
|
||||
class WorkerDocStorage implements DocStorage {
|
||||
constructor(
|
||||
private readonly client: OpClient<WorkerOps>,
|
||||
private readonly options: StorageOptions
|
||||
) {}
|
||||
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||
|
||||
readonly peer = this.options.peer;
|
||||
readonly spaceType = this.options.type;
|
||||
readonly spaceId = this.options.id;
|
||||
readonly universalId = universalId(this.options);
|
||||
readonly storageType = 'doc';
|
||||
readonly isReadonly = false;
|
||||
|
||||
async getDoc(docId: string) {
|
||||
return this.client.call('docStorage.getDoc', docId);
|
||||
@@ -119,16 +123,9 @@ class WorkerDocConnection extends DummyConnection {
|
||||
}
|
||||
|
||||
class WorkerBlobStorage implements BlobStorage {
|
||||
constructor(
|
||||
private readonly client: OpClient<WorkerOps>,
|
||||
private readonly options: StorageOptions
|
||||
) {}
|
||||
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||
|
||||
readonly storageType = 'blob';
|
||||
readonly peer = this.options.peer;
|
||||
readonly spaceType = this.options.type;
|
||||
readonly spaceId = this.options.id;
|
||||
readonly universalId = universalId(this.options);
|
||||
|
||||
get(key: string, _signal?: AbortSignal): Promise<BlobRecord | null> {
|
||||
return this.client.call('blobStorage.getBlob', key);
|
||||
@@ -156,63 +153,6 @@ class WorkerBlobStorage implements BlobStorage {
|
||||
connection = new DummyConnection();
|
||||
}
|
||||
|
||||
class WorkerAwarenessStorage implements AwarenessStorage {
|
||||
constructor(
|
||||
private readonly client: OpClient<WorkerOps>,
|
||||
private readonly options: StorageOptions
|
||||
) {}
|
||||
|
||||
readonly storageType = 'awareness';
|
||||
readonly peer = this.options.peer;
|
||||
readonly spaceType = this.options.type;
|
||||
readonly spaceId = this.options.id;
|
||||
readonly universalId = universalId(this.options);
|
||||
|
||||
update(record: AwarenessRecord, origin?: string): Promise<void> {
|
||||
return this.client.call('awarenessStorage.update', {
|
||||
awareness: record,
|
||||
origin,
|
||||
});
|
||||
}
|
||||
subscribeUpdate(
|
||||
id: string,
|
||||
onUpdate: (update: AwarenessRecord, origin?: string) => void,
|
||||
onCollect: () => Promise<AwarenessRecord | null>
|
||||
): () => void {
|
||||
const subscription = this.client
|
||||
.ob$('awarenessStorage.subscribeUpdate', id)
|
||||
.subscribe({
|
||||
next: update => {
|
||||
if (update.type === 'awareness-update') {
|
||||
onUpdate(update.awareness, update.origin);
|
||||
}
|
||||
if (update.type === 'awareness-collect') {
|
||||
onCollect()
|
||||
.then(record => {
|
||||
if (record) {
|
||||
this.client
|
||||
.call('awarenessStorage.collect', {
|
||||
awareness: record,
|
||||
collectId: update.collectId,
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('error feedback collected awareness', err);
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
console.error('error collecting awareness', err);
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
return () => {
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}
|
||||
connection = new DummyConnection();
|
||||
}
|
||||
|
||||
class WorkerDocSync implements DocSync {
|
||||
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||
|
||||
@@ -234,6 +174,22 @@ class WorkerDocSync implements DocSync {
|
||||
|
||||
class WorkerBlobSync implements BlobSync {
|
||||
constructor(private readonly client: OpClient<WorkerOps>) {}
|
||||
readonly state$ = this.client.ob$('blobSync.state');
|
||||
setMaxBlobSize(size: number): void {
|
||||
this.client.call('blobSync.setMaxBlobSize', size).catch(err => {
|
||||
console.error('error setting max blob size', err);
|
||||
});
|
||||
}
|
||||
onReachedMaxBlobSize(cb: (byteSize: number) => void): () => void {
|
||||
const subscription = this.client
|
||||
.ob$('blobSync.onReachedMaxBlobSize')
|
||||
.subscribe(byteSize => {
|
||||
cb(byteSize);
|
||||
});
|
||||
return () => {
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
}
|
||||
downloadBlob(
|
||||
blobId: string,
|
||||
_signal?: AbortSignal
|
||||
@@ -243,6 +199,27 @@ class WorkerBlobSync implements BlobSync {
|
||||
uploadBlob(blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
|
||||
return this.client.call('blobSync.uploadBlob', blob);
|
||||
}
|
||||
fullSync(signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const abortListener = () => {
|
||||
reject(signal?.reason);
|
||||
subscription.unsubscribe();
|
||||
};
|
||||
|
||||
signal?.addEventListener('abort', abortListener);
|
||||
|
||||
const subscription = this.client.ob$('blobSync.fullSync').subscribe({
|
||||
next() {
|
||||
signal?.removeEventListener('abort', abortListener);
|
||||
resolve();
|
||||
},
|
||||
error(err) {
|
||||
signal?.removeEventListener('abort', abortListener);
|
||||
reject(err);
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class WorkerAwarenessSync implements AwarenessSync {
|
||||
|
||||
@@ -1,27 +1,29 @@
|
||||
import type { OpConsumer } from '@toeverything/infra/op';
|
||||
import { Observable } from 'rxjs';
|
||||
|
||||
import { getAvailableStorageImplementations } from '../impls';
|
||||
import { SpaceStorage, type StorageOptions } from '../storage';
|
||||
import { type StorageConstructor } from '../impls';
|
||||
import { SpaceStorage } from '../storage';
|
||||
import type { AwarenessRecord } from '../storage/awareness';
|
||||
import { Sync } from '../sync';
|
||||
import type { WorkerOps } from './ops';
|
||||
import type { PeerStorageOptions } from '../sync/types';
|
||||
import type { WorkerInitOptions, WorkerOps } from './ops';
|
||||
|
||||
export type { WorkerOps };
|
||||
|
||||
export class WorkerConsumer {
|
||||
private remotes: SpaceStorage[] = [];
|
||||
private local: SpaceStorage | null = null;
|
||||
private storages: PeerStorageOptions<SpaceStorage> | null = null;
|
||||
private sync: Sync | null = null;
|
||||
|
||||
get ensureLocal() {
|
||||
if (!this.local) {
|
||||
if (!this.storages) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
return this.local;
|
||||
return this.storages.local;
|
||||
}
|
||||
|
||||
get ensureSync() {
|
||||
if (!this.sync) {
|
||||
throw new Error('Not initialized');
|
||||
throw new Error('Sync not initialized');
|
||||
}
|
||||
return this.sync;
|
||||
}
|
||||
@@ -31,11 +33,7 @@ export class WorkerConsumer {
|
||||
}
|
||||
|
||||
get docSync() {
|
||||
const docSync = this.ensureSync.doc;
|
||||
if (!docSync) {
|
||||
throw new Error('Doc sync not initialized');
|
||||
}
|
||||
return docSync;
|
||||
return this.ensureSync.doc;
|
||||
}
|
||||
|
||||
get blobStorage() {
|
||||
@@ -43,11 +41,7 @@ export class WorkerConsumer {
|
||||
}
|
||||
|
||||
get blobSync() {
|
||||
const blobSync = this.ensureSync.blob;
|
||||
if (!blobSync) {
|
||||
throw new Error('Blob sync not initialized');
|
||||
}
|
||||
return blobSync;
|
||||
return this.ensureSync.blob;
|
||||
}
|
||||
|
||||
get syncStorage() {
|
||||
@@ -59,41 +53,58 @@ export class WorkerConsumer {
|
||||
}
|
||||
|
||||
get awarenessSync() {
|
||||
const awarenessSync = this.ensureSync.awareness;
|
||||
if (!awarenessSync) {
|
||||
throw new Error('Awareness sync not initialized');
|
||||
}
|
||||
return awarenessSync;
|
||||
return this.ensureSync.awareness;
|
||||
}
|
||||
|
||||
constructor(private readonly consumer: OpConsumer<WorkerOps>) {}
|
||||
|
||||
listen() {
|
||||
constructor(
|
||||
private readonly consumer: OpConsumer<WorkerOps>,
|
||||
private readonly availableStorageImplementations: StorageConstructor[]
|
||||
) {
|
||||
this.registerHandlers();
|
||||
this.consumer.listen();
|
||||
}
|
||||
|
||||
async init(init: {
|
||||
local: { name: string; opts: StorageOptions }[];
|
||||
remotes: { name: string; opts: StorageOptions }[][];
|
||||
}) {
|
||||
this.local = new SpaceStorage(
|
||||
init.local.map(opt => {
|
||||
const Storage = getAvailableStorageImplementations(opt.name);
|
||||
return new Storage(opt.opts);
|
||||
})
|
||||
);
|
||||
this.remotes = init.remotes.map(opts => {
|
||||
return new SpaceStorage(
|
||||
opts.map(opt => {
|
||||
const Storage = getAvailableStorageImplementations(opt.name);
|
||||
return new Storage(opt.opts);
|
||||
init(init: WorkerInitOptions) {
|
||||
this.storages = {
|
||||
local: new SpaceStorage(
|
||||
Object.fromEntries(
|
||||
Object.entries(init.local).map(([type, opt]) => {
|
||||
const Storage = this.availableStorageImplementations.find(
|
||||
impl => impl.identifier === opt.name
|
||||
);
|
||||
if (!Storage) {
|
||||
throw new Error(`Storage implementation ${opt.name} not found`);
|
||||
}
|
||||
return [type, new Storage(opt.opts as any)];
|
||||
})
|
||||
)
|
||||
),
|
||||
remotes: Object.fromEntries(
|
||||
Object.entries(init.remotes).map(([peer, opts]) => {
|
||||
return [
|
||||
peer,
|
||||
new SpaceStorage(
|
||||
Object.fromEntries(
|
||||
Object.entries(opts).map(([type, opt]) => {
|
||||
const Storage = this.availableStorageImplementations.find(
|
||||
impl => impl.identifier === opt.name
|
||||
);
|
||||
if (!Storage) {
|
||||
throw new Error(
|
||||
`Storage implementation ${opt.name} not found`
|
||||
);
|
||||
}
|
||||
return [type, new Storage(opt.opts as any)];
|
||||
})
|
||||
)
|
||||
),
|
||||
];
|
||||
})
|
||||
);
|
||||
});
|
||||
this.sync = new Sync(this.local, this.remotes);
|
||||
this.local.connect();
|
||||
for (const remote of this.remotes) {
|
||||
),
|
||||
};
|
||||
this.sync = new Sync(this.storages);
|
||||
this.storages.local.connect();
|
||||
for (const remote of Object.values(this.storages.remotes)) {
|
||||
remote.connect();
|
||||
}
|
||||
this.sync.start();
|
||||
@@ -101,9 +112,9 @@ export class WorkerConsumer {
|
||||
|
||||
async destroy() {
|
||||
this.sync?.stop();
|
||||
this.local?.disconnect();
|
||||
await this.local?.destroy();
|
||||
for (const remote of this.remotes) {
|
||||
this.storages?.local.disconnect();
|
||||
await this.storages?.local.destroy();
|
||||
for (const remote of Object.values(this.storages?.remotes ?? {})) {
|
||||
remote.disconnect();
|
||||
await remote.destroy();
|
||||
}
|
||||
@@ -144,7 +155,7 @@ export class WorkerConsumer {
|
||||
subscriber.next(true);
|
||||
subscriber.complete();
|
||||
})
|
||||
.catch(error => {
|
||||
.catch((error: any) => {
|
||||
subscriber.error(error);
|
||||
});
|
||||
return () => abortController.abort();
|
||||
@@ -224,6 +235,29 @@ export class WorkerConsumer {
|
||||
}),
|
||||
'blobSync.downloadBlob': key => this.blobSync.downloadBlob(key),
|
||||
'blobSync.uploadBlob': blob => this.blobSync.uploadBlob(blob),
|
||||
'blobSync.fullSync': () =>
|
||||
new Observable(subscriber => {
|
||||
const abortController = new AbortController();
|
||||
this.blobSync
|
||||
.fullSync(abortController.signal)
|
||||
.then(() => {
|
||||
subscriber.next(true);
|
||||
subscriber.complete();
|
||||
})
|
||||
.catch(error => {
|
||||
subscriber.error(error);
|
||||
});
|
||||
return () => abortController.abort();
|
||||
}),
|
||||
'blobSync.state': () => this.blobSync.state$,
|
||||
'blobSync.setMaxBlobSize': size => this.blobSync.setMaxBlobSize(size),
|
||||
'blobSync.onReachedMaxBlobSize': () =>
|
||||
new Observable(subscriber => {
|
||||
const undo = this.blobSync.onReachedMaxBlobSize(byteSize => {
|
||||
subscriber.next(byteSize);
|
||||
});
|
||||
return () => undo();
|
||||
}),
|
||||
'awarenessSync.update': ({ awareness, origin }) =>
|
||||
this.awarenessSync.update(awareness, origin),
|
||||
'awarenessSync.subscribeUpdate': docId =>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { AvailableStorageImplementations } from '../impls';
|
||||
import type {
|
||||
BlobRecord,
|
||||
DocClock,
|
||||
@@ -6,20 +7,27 @@ import type {
|
||||
DocRecord,
|
||||
DocUpdate,
|
||||
ListedBlobRecord,
|
||||
StorageOptions,
|
||||
StorageType,
|
||||
} from '../storage';
|
||||
import type { AwarenessRecord } from '../storage/awareness';
|
||||
import type { BlobSyncState } from '../sync/blob';
|
||||
import type { DocSyncDocState, DocSyncState } from '../sync/doc';
|
||||
|
||||
type StorageInitOptions = Values<{
|
||||
[key in keyof AvailableStorageImplementations]: {
|
||||
name: key;
|
||||
opts: ConstructorParameters<AvailableStorageImplementations[key]>[0];
|
||||
};
|
||||
}>;
|
||||
|
||||
export interface WorkerInitOptions {
|
||||
local: { [key in StorageType]?: StorageInitOptions };
|
||||
remotes: Record<string, { [key in StorageType]?: StorageInitOptions }>;
|
||||
}
|
||||
|
||||
interface GroupedWorkerOps {
|
||||
worker: {
|
||||
init: [
|
||||
{
|
||||
local: { name: string; opts: StorageOptions }[];
|
||||
remotes: { name: string; opts: StorageOptions }[][];
|
||||
},
|
||||
void,
|
||||
];
|
||||
init: [WorkerInitOptions, void];
|
||||
destroy: [void, void];
|
||||
};
|
||||
|
||||
@@ -83,6 +91,10 @@ interface GroupedWorkerOps {
|
||||
blobSync: {
|
||||
downloadBlob: [string, BlobRecord | null];
|
||||
uploadBlob: [BlobRecord, void];
|
||||
fullSync: [void, boolean];
|
||||
setMaxBlobSize: [number, void];
|
||||
onReachedMaxBlobSize: [void, number];
|
||||
state: [void, BlobSyncState];
|
||||
};
|
||||
|
||||
awarenessSync: {
|
||||
|
||||
Reference in New Issue
Block a user