feat(nbstore): improve nbstore (#9512)

packages/common/nbstore/src/storage/__tests__/__snapshots__/storage.spec.ts.snap

@@ -1,41 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`parseUniversalId > should parse universal id > @peer(@name);@type(userspace);@id(@id); 1`] = `
-{
-  "id": "@id",
-  "peer": "@name",
-  "type": "userspace",
-}
-`;
-
-exports[`parseUniversalId > should parse universal id > @peer(@peer(name);@type(userspace);@id(@id); 1`] = `
-{
-  "id": "@id",
-  "peer": "@peer(name",
-  "type": "userspace",
-}
-`;
-
-exports[`parseUniversalId > should parse universal id > @peer(123);@type(userspace);@id(456); 1`] = `
-{
-  "id": "456",
-  "peer": "123",
-  "type": "userspace",
-}
-`;
-
-exports[`parseUniversalId > should parse universal id > @peer(123);@type(workspace);@id(456); 1`] = `
-{
-  "id": "456",
-  "peer": "123",
-  "type": "workspace",
-}
-`;
-
-exports[`parseUniversalId > should parse universal id > @peer(https://app.affine.pro);@type(userspace);@id(hello:world); 1`] = `
-{
-  "id": "hello:world",
-  "peer": "https://app.affine.pro",
-  "type": "userspace",
-}
-`;

packages/common/nbstore/src/storage/__tests__/storage.spec.ts

@@ -1,36 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import { parseUniversalId, universalId } from '../storage';
-
-describe('parseUniversalId', () => {
-  it('should generate universal id', () => {
-    expect(universalId({ peer: '123', type: 'workspace', id: '456' })).toEqual(
-      '@peer(123);@type(workspace);@id(456);'
-    );
-  });
-
-  it('should parse universal id', () => {
-    const testcases = [
-      '@peer(123);@type(userspace);@id(456);',
-      '@peer(123);@type(workspace);@id(456);',
-      '@peer(https://app.affine.pro);@type(userspace);@id(hello:world);',
-      '@peer(@name);@type(userspace);@id(@id);',
-      '@peer(@peer(name);@type(userspace);@id(@id);',
-    ];
-
-    testcases.forEach(id => {
-      expect(parseUniversalId(id)).toMatchSnapshot(id);
-    });
-  });
-
-  it('should throw invalid universal id', () => {
-    const testcases = [
-      '@peer(123);@type(anyspace);@id(456);', // invalid space type
-      '@peer(@peer(name););@type(userspace);@id(@id);', // invalid peer
-    ];
-
-    testcases.forEach(id => {
-      expect(() => parseUniversalId(id)).toThrow();
-    });
-  });
-});

packages/common/nbstore/src/storage/awareness.ts

@@ -1,6 +1,5 @@
-import { type Storage, StorageBase, type StorageOptions } from './storage';
-
-export interface AwarenessStorageOptions extends StorageOptions {}
+import type { Connection } from '../connection';
+import { type Storage } from './storage';
 
 export type AwarenessRecord = {
   docId: string;
@@ -23,13 +22,9 @@ export interface AwarenessStorage extends Storage {
   ): () => void;
 }
 
-export abstract class AwarenessStorageBase<
-    Options extends AwarenessStorageOptions = AwarenessStorageOptions,
-  >
-  extends StorageBase<Options>
-  implements AwarenessStorage
-{
-  override readonly storageType = 'awareness';
+export abstract class AwarenessStorageBase implements AwarenessStorage {
+  readonly storageType = 'awareness';
+  abstract readonly connection: Connection;
 
   abstract update(record: AwarenessRecord, origin?: string): Promise<void>;
 

packages/common/nbstore/src/storage/blob.ts

@@ -1,6 +1,5 @@
-import { type Storage, StorageBase, type StorageOptions } from './storage';
-
-export interface BlobStorageOptions extends StorageOptions {}
+import type { Connection } from '../connection';
+import { type Storage } from './storage';
 
 export interface BlobRecord {
   key: string;
@@ -29,13 +28,9 @@ export interface BlobStorage extends Storage {
   list(signal?: AbortSignal): Promise<ListedBlobRecord[]>;
 }
 
-export abstract class BlobStorageBase<
-    Options extends BlobStorageOptions = BlobStorageOptions,
-  >
-  extends StorageBase<Options>
-  implements BlobStorage
-{
-  override readonly storageType = 'blob';
+export abstract class BlobStorageBase implements BlobStorage {
+  readonly storageType = 'blob';
+  abstract readonly connection: Connection;
 
   abstract get(key: string, signal?: AbortSignal): Promise<BlobRecord | null>;
   abstract set(blob: BlobRecord, signal?: AbortSignal): Promise<void>;

packages/common/nbstore/src/storage/doc.ts

@@ -1,10 +1,11 @@
 import EventEmitter2 from 'eventemitter2';
 import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs';
 
+import type { Connection } from '../connection';
 import { isEmptyUpdate } from '../utils/is-empty-update';
 import type { Locker } from './lock';
 import { SingletonLocker } from './lock';
-import { type Storage, StorageBase, type StorageOptions } from './storage';
+import { type Storage } from './storage';
 
 export interface DocClock {
   docId: string;
@@ -33,13 +34,19 @@ export interface Editor {
   avatarUrl: string | null;
 }
 
-export interface DocStorageOptions extends StorageOptions {
+export interface DocStorageOptions {
   mergeUpdates?: (updates: Uint8Array[]) => Promise<Uint8Array> | Uint8Array;
+  id: string;
+
+  /**
+   * open as readonly mode.
+   */
+  readonlyMode?: boolean;
 }
 
 export interface DocStorage extends Storage {
   readonly storageType: 'doc';
-
+  readonly isReadonly: boolean;
   /**
    * Get a doc record with latest binary.
    */
@@ -88,18 +95,22 @@ export interface DocStorage extends Storage {
   ): () => void;
 }
 
-export abstract class DocStorageBase<
-    Opts extends DocStorageOptions = DocStorageOptions,
-  >
-  extends StorageBase<Opts>
-  implements DocStorage
-{
+export abstract class DocStorageBase<Opts = {}> implements DocStorage {
+  get isReadonly(): boolean {
+    return this.options.readonlyMode ?? false;
+  }
   private readonly event = new EventEmitter2();
-  override readonly storageType = 'doc';
+  readonly storageType = 'doc';
+  abstract readonly connection: Connection;
   protected readonly locker: Locker = new SingletonLocker();
+  protected readonly spaceId = this.options.id;
+
+  constructor(protected readonly options: Opts & DocStorageOptions) {}
 
   async getDoc(docId: string) {
-    await using _lock = await this.lockDocForUpdate(docId);
+    await using _lock = this.isReadonly
+      ? undefined
+      : await this.lockDocForUpdate(docId);
 
     const snapshot = await this.getDocSnapshot(docId);
     const updates = await this.getDocUpdates(docId);
@@ -117,10 +128,13 @@ export abstract class DocStorageBase<
         editor,
       };
 
-      await this.setDocSnapshot(newSnapshot, snapshot);
+      // if is readonly, we will not set the new snapshot
+      if (!this.isReadonly) {
+        await this.setDocSnapshot(newSnapshot, snapshot);
 
-      // always mark updates as merged unless throws
-      await this.markUpdatesMerged(docId, updates);
+        // always mark updates as merged unless throws
+        await this.markUpdatesMerged(docId, updates);
+      }
 
       return newSnapshot;
     }

packages/common/nbstore/src/storage/dummy/awareness.ts

@@ -0,0 +1,16 @@
+import { DummyConnection } from '../../connection';
+import { type AwarenessRecord, AwarenessStorageBase } from '../awareness';
+
+export class DummyAwarenessStorage extends AwarenessStorageBase {
+  override update(_record: AwarenessRecord, _origin?: string): Promise<void> {
+    return Promise.resolve();
+  }
+  override subscribeUpdate(
+    _id: string,
+    _onUpdate: (update: AwarenessRecord, origin?: string) => void,
+    _onCollect: () => Promise<AwarenessRecord | null>
+  ): () => void {
+    return () => {};
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/blob.ts

@@ -0,0 +1,32 @@
+import { DummyConnection } from '../../connection';
+import {
+  type BlobRecord,
+  BlobStorageBase,
+  type ListedBlobRecord,
+} from '../blob';
+
+export class DummyBlobStorage extends BlobStorageBase {
+  override get(
+    _key: string,
+    _signal?: AbortSignal
+  ): Promise<BlobRecord | null> {
+    return Promise.resolve(null);
+  }
+  override set(_blob: BlobRecord, _signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  override delete(
+    _key: string,
+    _permanently: boolean,
+    _signal?: AbortSignal
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override release(_signal?: AbortSignal): Promise<void> {
+    return Promise.resolve();
+  }
+  override list(_signal?: AbortSignal): Promise<ListedBlobRecord[]> {
+    return Promise.resolve([]);
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/doc.ts

@@ -0,0 +1,41 @@
+import { DummyConnection } from '../../connection';
+import {
+  type DocClock,
+  type DocClocks,
+  type DocDiff,
+  type DocRecord,
+  type DocStorage,
+  type DocUpdate,
+} from '../doc';
+
+export class DummyDocStorage implements DocStorage {
+  readonly storageType = 'doc';
+  readonly isReadonly = true;
+  getDoc(_docId: string): Promise<DocRecord | null> {
+    return Promise.resolve(null);
+  }
+  getDocDiff(_docId: string, _state?: Uint8Array): Promise<DocDiff | null> {
+    return Promise.resolve(null);
+  }
+  pushDocUpdate(update: DocUpdate, _origin?: string): Promise<DocClock> {
+    return Promise.resolve({
+      docId: update.docId,
+      timestamp: new Date(),
+    });
+  }
+  getDocTimestamp(_docId: string): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  getDocTimestamps(_after?: Date): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  deleteDoc(_docId: string): Promise<void> {
+    return Promise.resolve();
+  }
+  subscribeDocUpdate(
+    _callback: (update: DocRecord, origin?: string) => void
+  ): () => void {
+    return () => {};
+  }
+  connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/dummy/sync.ts

@@ -0,0 +1,49 @@
+import { DummyConnection } from '../../connection';
+import type { DocClock, DocClocks } from '../doc';
+import { SyncStorageBase } from '../sync';
+
+export class DummySyncStorage extends SyncStorageBase {
+  override getPeerRemoteClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerRemoteClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerRemoteClock(_peer: string, _clock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override getPeerPulledRemoteClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerPulledRemoteClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerPulledRemoteClock(
+    _peer: string,
+    _clock: DocClock
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override getPeerPushedClock(
+    _peer: string,
+    _docId: string
+  ): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override getPeerPushedClocks(_peer: string): Promise<DocClocks> {
+    return Promise.resolve({});
+  }
+  override setPeerPushedClock(_peer: string, _clock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override clearClocks(): Promise<void> {
+    return Promise.resolve();
+  }
+  override connection = new DummyConnection();
+}

packages/common/nbstore/src/storage/errors/index.ts

@@ -0,0 +1 @@
+export * from './over-capacity';

packages/common/nbstore/src/storage/errors/over-capacity.ts

@@ -0,0 +1,5 @@
+export class OverCapacityError extends Error {
+  constructor(public originError?: any) {
+    super('Storage over capacity. Origin error: ' + originError);
+  }
+}

packages/common/nbstore/src/storage/index.ts

@@ -3,56 +3,60 @@ import EventEmitter2 from 'eventemitter2';
 import type { AwarenessStorage } from './awareness';
 import type { BlobStorage } from './blob';
 import type { DocStorage } from './doc';
-import type { Storage, StorageType } from './storage';
+import { DummyAwarenessStorage } from './dummy/awareness';
+import { DummyBlobStorage } from './dummy/blob';
+import { DummyDocStorage } from './dummy/doc';
+import { DummySyncStorage } from './dummy/sync';
+import type { StorageType } from './storage';
 import type { SyncStorage } from './sync';
 
 type Storages = DocStorage | BlobStorage | SyncStorage | AwarenessStorage;
 
+export type SpaceStorageOptions = {
+  [K in StorageType]?: Storages & { storageType: K };
+};
+
 export class SpaceStorage {
-  protected readonly storages: Map<StorageType, Storage> = new Map();
+  protected readonly storages: {
+    [K in StorageType]: Storages & { storageType: K };
+  };
   private readonly event = new EventEmitter2();
   private readonly disposables: Set<() => void> = new Set();
 
-  constructor(storages: Storage[] = []) {
-    this.storages = new Map(
-      storages.map(storage => [storage.storageType, storage])
-    );
-  }
-
-  tryGet<T extends StorageType>(
-    type: T
-  ): Extract<Storages, { storageType: T }> | undefined {
-    return this.storages.get(type) as unknown as Extract<
-      Storages,
-      { storageType: T }
-    >;
+  constructor(storages: SpaceStorageOptions) {
+    this.storages = {
+      awareness: storages.awareness ?? new DummyAwarenessStorage(),
+      blob: storages.blob ?? new DummyBlobStorage(),
+      doc: storages.doc ?? new DummyDocStorage(),
+      sync: storages.sync ?? new DummySyncStorage(),
+    };
   }
 
   get<T extends StorageType>(type: T): Extract<Storages, { storageType: T }> {
-    const storage = this.tryGet(type);
+    const storage = this.storages[type];
 
     if (!storage) {
       throw new Error(`Storage ${type} not registered.`);
     }
 
-    return storage as Extract<Storages, { storageType: T }>;
+    return storage as unknown as Extract<Storages, { storageType: T }>;
   }
 
   connect() {
-    Array.from(this.storages.values()).forEach(storage => {
+    Object.values(this.storages).forEach(storage => {
       storage.connection.connect();
     });
   }
 
   disconnect() {
-    Array.from(this.storages.values()).forEach(storage => {
+    Object.values(this.storages).forEach(storage => {
       storage.connection.disconnect();
     });
   }
 
   async waitForConnected(signal?: AbortSignal) {
     await Promise.all(
-      Array.from(this.storages.values()).map(storage =>
+      Object.values(this.storages).map(storage =>
         storage.connection.waitForConnected(signal)
       )
     );
@@ -61,13 +65,13 @@ export class SpaceStorage {
   async destroy() {
     this.disposables.forEach(disposable => disposable());
     this.event.removeAllListeners();
-    this.storages.clear();
   }
 }
 
 export * from './awareness';
 export * from './blob';
 export * from './doc';
+export * from './errors';
 export * from './history';
 export * from './storage';
 export * from './sync';

packages/common/nbstore/src/storage/storage.ts

@@ -1,120 +1,8 @@
 import type { Connection } from '../connection';
 
-export type SpaceType = 'workspace' | 'userspace';
 export type StorageType = 'blob' | 'doc' | 'sync' | 'awareness';
 
-export interface StorageOptions {
-  peer: string;
-  type: SpaceType;
-  id: string;
-}
-
-export function universalId({ peer, type, id }: StorageOptions) {
-  return `@peer(${peer});@type(${type});@id(${id});`;
-}
-
-export function isValidSpaceType(type: string): type is SpaceType {
-  return type === 'workspace' || type === 'userspace';
-}
-
-export function isValidUniversalId(opts: Record<string, string>): boolean {
-  const requiredKeys: Array<keyof StorageOptions> = [
-    'peer',
-    'type',
-    'id',
-  ] as const;
-
-  for (const key of requiredKeys) {
-    if (!opts[key]) {
-      return false;
-    }
-  }
-
-  return isValidSpaceType(opts.type);
-}
-
-export function parseUniversalId(id: string) {
-  const result: Partial<StorageOptions> = {};
-  let key = '';
-  let value = '';
-  let isInValue = false;
-
-  let i = -1;
-
-  while (++i < id.length) {
-    const ch = id[i];
-    const nextCh = id[i + 1];
-
-    // when we are in value string, we only care about ch and next char to be [')', ';'] to end the id part
-    if (isInValue) {
-      if (ch === ')' && nextCh === ';') {
-        // @ts-expect-error we know the key is valid
-        result[key] = value;
-        key = '';
-        value = '';
-        isInValue = false;
-        i++;
-        continue;
-      }
-
-      value += ch;
-      continue;
-    }
-
-    if (ch === '@') {
-      const keyEnd = id.indexOf('(', i);
-      // we find '@' but no '(' in lookahead or '(' is immediately after '@', invalid id
-      if (keyEnd === -1 || keyEnd === i + 1) {
-        break;
-      }
-
-      key = id.slice(i + 1, keyEnd);
-      i = keyEnd;
-      isInValue = true;
-    } else {
-      break;
-    }
-  }
-
-  if (!isValidUniversalId(result)) {
-    throw new Error(
-      `Invalid universal storage id: ${id}. It should be in format of @peer(\${peer});@type(\${type});@id(\${id});`
-    );
-  }
-
-  return result as StorageOptions;
-}
-
 export interface Storage {
   readonly storageType: StorageType;
   readonly connection: Connection;
-  readonly peer: string;
-  readonly spaceType: string;
-  readonly spaceId: string;
-  readonly universalId: string;
-}
-
-export abstract class StorageBase<Opts extends StorageOptions = StorageOptions>
-  implements Storage
-{
-  abstract readonly storageType: StorageType;
-  abstract readonly connection: Connection;
-
-  get peer() {
-    return this.options.peer;
-  }
-
-  get spaceType() {
-    return this.options.type;
-  }
-
-  get spaceId() {
-    return this.options.id;
-  }
-
-  get universalId() {
-    return universalId(this.options);
-  }
-
-  constructor(public readonly options: Opts) {}
 }

packages/common/nbstore/src/storage/sync.ts

@@ -1,7 +1,6 @@
+import type { Connection } from '../connection';
 import type { DocClock, DocClocks } from './doc';
-import { type Storage, StorageBase, type StorageOptions } from './storage';
-
-export interface SyncStorageOptions extends StorageOptions {}
+import { type Storage } from './storage';
 
 export interface SyncStorage extends Storage {
   readonly storageType: 'sync';
@@ -21,13 +20,9 @@ export interface SyncStorage extends Storage {
   clearClocks(): Promise<void>;
 }
 
-export abstract class BasicSyncStorage<
-    Opts extends SyncStorageOptions = SyncStorageOptions,
-  >
-  extends StorageBase<Opts>
-  implements SyncStorage
-{
-  override readonly storageType = 'sync';
+export abstract class SyncStorageBase implements SyncStorage {
+  readonly storageType = 'sync';
+  abstract readonly connection: Connection;
 
   abstract getPeerRemoteClock(
     peer: string,