feat(nbstore): add nbstore worker (#9185)

packages/common/nbstore/src/storage/awareness.ts

@@ -1,4 +1,4 @@
-import { Storage, type StorageOptions } from './storage';
+import { type Storage, StorageBase, type StorageOptions } from './storage';
 
 export interface AwarenessStorageOptions extends StorageOptions {}
 
@@ -7,21 +7,35 @@ export type AwarenessRecord = {
   bin: Uint8Array;
 };
 
-export abstract class AwarenessStorage<
-  Options extends AwarenessStorageOptions = AwarenessStorageOptions,
-> extends Storage<Options> {
-  override readonly storageType = 'awareness';
+export interface AwarenessStorage extends Storage {
+  readonly storageType: 'awareness';
 
   /**
    * Update the awareness record.
    *
    * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
    */
+  update(record: AwarenessRecord, origin?: string): Promise<void>;
+  subscribeUpdate(
+    id: string,
+    onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void;
+}
+
+export abstract class AwarenessStorageBase<
+    Options extends AwarenessStorageOptions = AwarenessStorageOptions,
+  >
+  extends StorageBase<Options>
+  implements AwarenessStorage
+{
+  override readonly storageType = 'awareness';
+
   abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
 
   abstract subscribeUpdate(
     id: string,
     onUpdate: (update: AwarenessRecord, origin?: string) => void,
-    onCollect: () => AwarenessRecord
+    onCollect: () => Promise<AwarenessRecord | null>
   ): () => void;
 }

packages/common/nbstore/src/storage/blob.ts

@@ -1,4 +1,4 @@
-import { Storage, type StorageOptions } from './storage';
+import { type Storage, StorageBase, type StorageOptions } from './storage';
 
 export interface BlobStorageOptions extends StorageOptions {}
 
@@ -16,9 +16,25 @@ export interface ListedBlobRecord {
   createdAt?: Date;
 }
 
-export abstract class BlobStorage<
-  Options extends BlobStorageOptions = BlobStorageOptions,
-> extends Storage<Options> {
+export interface BlobStorage extends Storage {
+  readonly storageType: 'blob';
+  get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;
+  set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;
+  delete(
+    key: string,
+    permanently: boolean,
+    signal?: AbortSignal
+  ): Promise<void>;
+  release(signal?: AbortSignal): Promise<void>;
+  list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
+}
+
+export abstract class BlobStorageBase<
+    Options extends BlobStorageOptions = BlobStorageOptions,
+  >
+  extends StorageBase<Options>
+  implements BlobStorage
+{
   override readonly storageType = 'blob';
 
   abstract get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;

packages/common/nbstore/src/storage/doc.ts

@@ -4,7 +4,7 @@ import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs';
 import { isEmptyUpdate } from '../utils/is-empty-update';
 import type { Locker } from './lock';
 import { SingletonLocker } from './lock';
-import { Storage, type StorageOptions } from './storage';
+import { type Storage, StorageBase, type StorageOptions } from './storage';
 
 export interface DocClock {
   docId: string;
@@ -37,17 +37,67 @@ export interface DocStorageOptions extends StorageOptions {
   mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
 }
 
-export abstract class DocStorage<
-  Opts extends DocStorageOptions = DocStorageOptions,
-> extends Storage<Opts> {
+export interface DocStorage extends Storage {
+  readonly storageType: 'doc';
+
+  /**
+   * Get a doc record with latest binary.
+   */
+  getDoc(docId: string): Promise<DocRecord | null>;
+  /**
+   * Get a yjs binary diff with the given state vector.
+   */
+  getDocDiff(docId: string, state?: Uint8Array): Promise<DocDiff | null>;
+  /**
+   * Push updates into storage
+   *
+   * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
+   */
+  pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
+
+  /**
+   * Get the timestamp of the latest update of a doc.
+   */
+  getDocTimestamp(docId: string): Promise<DocClock | null>;
+
+  /**
+   * Get all docs timestamps info. especially for useful in sync process.
+   */
+  getDocTimestamps(after?: Date): Promise<DocClocks>;
+
+  /**
+   * Delete a specific doc data with all snapshots and updates
+   */
+  deleteDoc(docId: string): Promise<void>;
+
+  /**
+   * Subscribe on doc updates emitted from storage itself.
+   *
+   * NOTE:
+   *
+   *   There is not always update emitted from storage itself.
+   *
+   *   For example, in Sqlite storage, the update will only come from user's updating on docs,
+   *   in other words, the update will never somehow auto generated in storage internally.
+   *
+   *   But for Cloud storage, there will be updates broadcasted from other clients,
+   *   so the storage will emit updates to notify the client to integrate them.
+   */
+  subscribeDocUpdate(
+    callback: (update: DocRecord, origin?: string) => void
+  ): () => void;
+}
+
+export abstract class DocStorageBase<
+    Opts extends DocStorageOptions = DocStorageOptions,
+  >
+  extends StorageBase<Opts>
+  implements DocStorage
+{
   private readonly event = new EventEmitter2();
   override readonly storageType = 'doc';
   protected readonly locker: Locker = new SingletonLocker();
 
-  // REGION: open apis by Op system
-  /**
-   * Get a doc record with latest binary.
-   */
   async getDoc(docId: string) {
     await using _lock = await this.lockDocForUpdate(docId);
 
@@ -78,9 +128,6 @@ export abstract class DocStorage<
     return snapshot;
   }
 
-  /**
-   * Get a yjs binary diff with the given state vector.
-   */
   async getDocDiff(docId: string, state?: Uint8Array) {
     const doc = await this.getDoc(docId);
 
@@ -96,41 +143,14 @@ export abstract class DocStorage<
     };
   }
 
-  /**
-   * Push updates into storage
-   *
-   * @param origin - Internal identifier to recognize the source in the "update" event. Will not be stored or transferred.
-   */
   abstract pushDocUpdate(update: DocUpdate, origin?: string): Promise<DocClock>;
 
-  /**
-   * Get the timestamp of the latest update of a doc.
-   */
   abstract getDocTimestamp(docId: string): Promise<DocClock | null>;
 
-  /**
-   * Get all docs timestamps info. especially for useful in sync process.
-   */
   abstract getDocTimestamps(after?: Date): Promise<DocClocks>;
 
-  /**
-   * Delete a specific doc data with all snapshots and updates
-   */
   abstract deleteDoc(docId: string): Promise<void>;
 
-  /**
-   * Subscribe on doc updates emitted from storage itself.
-   *
-   * NOTE:
-   *
-   *   There is not always update emitted from storage itself.
-   *
-   *   For example, in Sqlite storage, the update will only come from user's updating on docs,
-   *   in other words, the update will never somehow auto generated in storage internally.
-   *
-   *   But for Cloud storage, there will be updates broadcasted from other clients,
-   *   so the storage will emit updates to notify the client to integrate them.
-   */
   subscribeDocUpdate(callback: (update: DocRecord, origin?: string) => void) {
     this.event.on('update', callback);
 
@@ -138,7 +158,6 @@ export abstract class DocStorage<
       this.event.off('update', callback);
     };
   }
-  // ENDREGION
 
   // REGION: api for internal usage
   protected on(

packages/common/nbstore/src/storage/history.ts

@@ -7,7 +7,7 @@ import {
   UndoManager,
 } from 'yjs';
 
-import { type DocRecord, DocStorage, type DocStorageOptions } from './doc';
+import { type DocRecord, DocStorageBase, type DocStorageOptions } from './doc';
 
 export interface HistoryFilter {
   before?: Date;
@@ -21,7 +21,7 @@ export interface ListedHistory {
 
 export abstract class HistoricalDocStorage<
   Options extends DocStorageOptions = DocStorageOptions,
-> extends DocStorage<Options> {
+> extends DocStorageBase<Options> {
   constructor(opts: Options) {
     super(opts);
 

packages/common/nbstore/src/storage/index.ts

@@ -40,20 +40,20 @@ export class SpaceStorage {
 
   connect() {
     Array.from(this.storages.values()).forEach(storage => {
-      storage.connect();
+      storage.connection.connect();
     });
   }
 
   disconnect() {
     Array.from(this.storages.values()).forEach(storage => {
-      storage.disconnect();
+      storage.connection.disconnect();
     });
   }
 
-  async waitForConnected() {
+  async waitForConnected(signal?: AbortSignal) {
     await Promise.all(
       Array.from(this.storages.values()).map(storage =>
-        storage.waitForConnected()
+        storage.connection.waitForConnected(signal)
       )
     );
   }
@@ -65,6 +65,7 @@ export class SpaceStorage {
   }
 }
 
+export * from './awareness';
 export * from './blob';
 export * from './doc';
 export * from './history';

packages/common/nbstore/src/storage/storage.ts

@@ -80,7 +80,18 @@ export function parseUniversalId(id: string) {
   return result as any;
 }
 
-export abstract class Storage<Opts extends StorageOptions = StorageOptions> {
+export interface Storage {
+  readonly storageType: StorageType;
+  readonly connection: Connection;
+  readonly peer: string;
+  readonly spaceType: string;
+  readonly spaceId: string;
+  readonly universalId: string;
+}
+
+export abstract class StorageBase<Opts extends StorageOptions = StorageOptions>
+  implements Storage
+{
   abstract readonly storageType: StorageType;
   abstract readonly connection: Connection;
 
@@ -101,16 +112,4 @@ export abstract class Storage<Opts extends StorageOptions = StorageOptions> {
   }
 
   constructor(public readonly options: Opts) {}
-
-  connect() {
-    this.connection.connect();
-  }
-
-  disconnect() {
-    this.connection.disconnect();
-  }
-
-  async waitForConnected() {
-    await this.connection.waitForConnected();
-  }
 }

packages/common/nbstore/src/storage/sync.ts

@@ -1,11 +1,32 @@
 import type { DocClock, DocClocks } from './doc';
-import { Storage, type StorageOptions } from './storage';
+import { type Storage, StorageBase, type StorageOptions } from './storage';
 
 export interface SyncStorageOptions extends StorageOptions {}
 
-export abstract class SyncStorage<
-  Opts extends SyncStorageOptions = SyncStorageOptions,
-> extends Storage<Opts> {
+export interface SyncStorage extends Storage {
+  readonly storageType: 'sync';
+
+  getPeerRemoteClock(peer: string, docId: string): Promise<DocClock | null>;
+  getPeerRemoteClocks(peer: string): Promise<DocClocks>;
+  setPeerRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  getPeerPulledRemoteClock(
+    peer: string,
+    docId: string
+  ): Promise<DocClock | null>;
+  getPeerPulledRemoteClocks(peer: string): Promise<DocClocks>;
+  setPeerPulledRemoteClock(peer: string, clock: DocClock): Promise<void>;
+  getPeerPushedClock(peer: string, docId: string): Promise<DocClock | null>;
+  getPeerPushedClocks(peer: string): Promise<DocClocks>;
+  setPeerPushedClock(peer: string, clock: DocClock): Promise<void>;
+  clearClocks(): Promise<void>;
+}
+
+export abstract class BasicSyncStorage<
+    Opts extends SyncStorageOptions = SyncStorageOptions,
+  >
+  extends StorageBase<Opts>
+  implements SyncStorage
+{
   override readonly storageType = 'sync';
 
   abstract getPeerRemoteClock(