feat(nbstore): improve nbstore (#9512)

packages/common/nbstore/src/impls/cloud/awareness.ts

@@ -4,21 +4,33 @@ import { share } from '../../connection';
 import {
   type AwarenessRecord,
   AwarenessStorageBase,
-  type AwarenessStorageOptions,
 } from '../../storage/awareness';
+import type { SpaceType } from '../../utils/universal-id';
 import {
   base64ToUint8Array,
   SocketConnection,
   uint8ArrayToBase64,
 } from './socket';
 
-interface CloudAwarenessStorageOptions extends AwarenessStorageOptions {
-  socketOptions: SocketOptions;
+interface CloudAwarenessStorageOptions {
+  socketOptions?: SocketOptions;
+  serverBaseUrl: string;
+  type: SpaceType;
+  id: string;
 }
 
-export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessStorageOptions> {
+export class CloudAwarenessStorage extends AwarenessStorageBase {
+  static readonly identifier = 'CloudAwarenessStorage';
+
+  constructor(private readonly options: CloudAwarenessStorageOptions) {
+    super();
+  }
+
   connection = share(
-    new SocketConnection(this.peer, this.options.socketOptions)
+    new SocketConnection(
+      `${this.options.serverBaseUrl}/`,
+      this.options.socketOptions
+    )
   );
 
   private get socket() {
@@ -28,8 +40,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessSt
   override async update(record: AwarenessRecord): Promise<void> {
     const encodedUpdate = await uint8ArrayToBase64(record.bin);
     this.socket.emit('space:update-awareness', {
-      spaceType: this.spaceType,
-      spaceId: this.spaceId,
+      spaceType: this.options.type,
+      spaceId: this.options.id,
       docId: record.docId,
       awarenessUpdate: encodedUpdate,
     });
@@ -44,8 +56,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessSt
     // leave awareness
     const leave = () => {
       this.socket.emit('space:leave-awareness', {
-        spaceType: this.spaceType,
-        spaceId: this.spaceId,
+        spaceType: this.options.type,
+        spaceId: this.options.id,
         docId: id,
       });
     };
@@ -53,14 +65,14 @@ export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessSt
     // join awareness, and collect awareness from others
     const joinAndCollect = async () => {
       await this.socket.emitWithAck('space:join-awareness', {
-        spaceType: this.spaceType,
-        spaceId: this.spaceId,
+        spaceType: this.options.type,
+        spaceId: this.options.id,
         docId: id,
         clientVersion: BUILD_CONFIG.appVersion,
       });
       this.socket.emit('space:load-awarenesses', {
-        spaceType: this.spaceType,
-        spaceId: this.spaceId,
+        spaceType: this.options.type,
+        spaceId: this.options.id,
         docId: id,
       });
     };
@@ -87,8 +99,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessSt
       docId: string;
     }) => {
       if (
-        spaceId === this.spaceId &&
-        spaceType === this.spaceType &&
+        spaceId === this.options.id &&
+        spaceType === this.options.type &&
         docId === id
       ) {
         (async () => {
@@ -96,8 +108,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessSt
           if (record) {
             const encodedUpdate = await uint8ArrayToBase64(record.bin);
             this.socket.emit('space:update-awareness', {
-              spaceType: this.spaceType,
-              spaceId: this.spaceId,
+              spaceType: this.options.type,
+              spaceId: this.options.id,
               docId: record.docId,
               awarenessUpdate: encodedUpdate,
             });
@@ -118,8 +130,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase<CloudAwarenessSt
       awarenessUpdate: string;
     }) => {
       if (
-        spaceId === this.spaceId &&
-        spaceType === this.spaceType &&
+        spaceId === this.options.id &&
+        spaceType === this.options.type &&
         docId === id
       ) {
         onUpdate({

packages/common/nbstore/src/impls/cloud/blob.ts

@@ -1,35 +1,30 @@
 import {
   deleteBlobMutation,
-  gqlFetcherFactory,
   listBlobsQuery,
   releaseDeletedBlobsMutation,
   setBlobMutation,
 } from '@affine/graphql';
 
-import { DummyConnection } from '../../connection';
-import {
-  type BlobRecord,
-  BlobStorageBase,
-  type BlobStorageOptions,
-} from '../../storage';
+import { type BlobRecord, BlobStorageBase } from '../../storage';
+import { HttpConnection } from './http';
 
-interface CloudBlobStorageOptions extends BlobStorageOptions {
-  apiBaseUrl: string;
+interface CloudBlobStorageOptions {
+  serverBaseUrl: string;
+  id: string;
 }
 
-export class CloudBlobStorage extends BlobStorageBase<CloudBlobStorageOptions> {
-  private readonly gql = gqlFetcherFactory(
-    this.options.apiBaseUrl + '/graphql'
-  );
-  override connection = new DummyConnection();
+export class CloudBlobStorage extends BlobStorageBase {
+  static readonly identifier = 'CloudBlobStorage';
+
+  constructor(private readonly options: CloudBlobStorageOptions) {
+    super();
+  }
+
+  readonly connection = new HttpConnection(this.options.serverBaseUrl);
 
   override async get(key: string) {
-    const res = await fetch(
-      this.options.apiBaseUrl +
-        '/api/workspaces/' +
-        this.spaceId +
-        '/blobs/' +
-        key,
+    const res = await this.connection.fetch(
+      '/api/workspaces/' + this.options.id + '/blobs/' + key,
       {
         cache: 'default',
         headers: {
@@ -38,49 +33,53 @@ export class CloudBlobStorage extends BlobStorageBase<CloudBlobStorageOptions> {
       }
     );
 
-    if (!res.ok) {
+    if (res.status === 404) {
       return null;
     }
 
-    const data = await res.arrayBuffer();
+    try {
+      const blob = await res.blob();
 
-    return {
-      key,
-      data: new Uint8Array(data),
-      mime: res.headers.get('content-type') || '',
-      size: data.byteLength,
-      createdAt: new Date(res.headers.get('last-modified') || Date.now()),
-    };
+      return {
+        key,
+        data: new Uint8Array(await blob.arrayBuffer()),
+        mime: blob.type,
+        size: blob.size,
+        createdAt: new Date(res.headers.get('last-modified') || Date.now()),
+      };
+    } catch (err) {
+      throw new Error('blob download error: ' + err);
+    }
   }
 
   override async set(blob: BlobRecord) {
-    await this.gql({
+    await this.connection.gql({
       query: setBlobMutation,
       variables: {
-        workspaceId: this.spaceId,
+        workspaceId: this.options.id,
         blob: new File([blob.data], blob.key, { type: blob.mime }),
       },
     });
   }
 
   override async delete(key: string, permanently: boolean) {
-    await this.gql({
+    await this.connection.gql({
       query: deleteBlobMutation,
-      variables: { workspaceId: this.spaceId, key, permanently },
+      variables: { workspaceId: this.options.id, key, permanently },
     });
   }
 
   override async release() {
-    await this.gql({
+    await this.connection.gql({
       query: releaseDeletedBlobsMutation,
-      variables: { workspaceId: this.spaceId },
+      variables: { workspaceId: this.options.id },
     });
   }
 
   override async list() {
-    const res = await this.gql({
+    const res = await this.connection.gql({
       query: listBlobsQuery,
-      variables: { workspaceId: this.spaceId },
+      variables: { workspaceId: this.options.id },
     });
 
     return res.workspace.blobs.map(blob => ({

packages/common/nbstore/src/impls/cloud/doc-static.ts

@@ -0,0 +1,82 @@
+import {
+  type DocClock,
+  type DocClocks,
+  type DocRecord,
+  DocStorageBase,
+  type DocStorageOptions,
+  type DocUpdate,
+} from '../../storage';
+import { HttpConnection } from './http';
+
+interface CloudDocStorageOptions extends DocStorageOptions {
+  serverBaseUrl: string;
+}
+
+export class StaticCloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
+  static readonly identifier = 'StaticCloudDocStorage';
+
+  constructor(options: CloudDocStorageOptions) {
+    super({ ...options, readonlyMode: true });
+  }
+
+  override connection = new HttpConnection(this.options.serverBaseUrl);
+  override async pushDocUpdate(
+    update: DocUpdate,
+    _origin?: string
+  ): Promise<DocClock> {
+    // http is readonly
+    return { docId: update.docId, timestamp: new Date() };
+  }
+  override async getDocTimestamp(docId: string): Promise<DocClock | null> {
+    // http doesn't support this, so we just return a new timestamp
+    return {
+      docId,
+      timestamp: new Date(),
+    };
+  }
+  override async getDocTimestamps(): Promise<DocClocks> {
+    // http doesn't support this
+    return {};
+  }
+  override deleteDoc(_docId: string): Promise<void> {
+    // http is readonly
+    return Promise.resolve();
+  }
+  protected override async getDocSnapshot(
+    docId: string
+  ): Promise<DocRecord | null> {
+    const arrayBuffer = await this.connection.fetchArrayBuffer(
+      `/api/workspaces/${this.spaceId}/docs/${docId}`,
+      {
+        priority: 'high',
+        headers: {
+          Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer
+        },
+      }
+    );
+    if (!arrayBuffer) {
+      return null;
+    }
+    return {
+      docId: docId,
+      bin: new Uint8Array(arrayBuffer),
+      timestamp: new Date(),
+    };
+  }
+  protected override setDocSnapshot(
+    _snapshot: DocRecord,
+    _prevSnapshot: DocRecord | null
+  ): Promise<boolean> {
+    // http is readonly
+    return Promise.resolve(false);
+  }
+  protected override getDocUpdates(_docId: string): Promise<DocRecord[]> {
+    return Promise.resolve([]);
+  }
+  protected override markUpdatesMerged(
+    _docId: string,
+    _updates: DocRecord[]
+  ): Promise<number> {
+    return Promise.resolve(0);
+  }
+}

packages/common/nbstore/src/impls/cloud/doc.ts

@@ -12,6 +12,7 @@ import {
   type DocStorageOptions,
   type DocUpdate,
 } from '../../storage';
+import type { SpaceType } from '../../utils/universal-id';
 import {
   base64ToUint8Array,
   type ServerEventsMap,
@@ -20,15 +21,20 @@ import {
 } from './socket';
 
 interface CloudDocStorageOptions extends DocStorageOptions {
-  socketOptions: SocketOptions;
+  socketOptions?: SocketOptions;
   serverBaseUrl: string;
+  type: SpaceType;
 }
 
 export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
+  static readonly identifier = 'CloudDocStorage';
+
   get socket() {
     return this.connection.inner;
   }
 
+  readonly spaceType = this.options.type;
+
   onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {
     if (
       this.spaceType === message.spaceType &&

packages/common/nbstore/src/impls/cloud/http.ts

@@ -0,0 +1,69 @@
+import { gqlFetcherFactory } from '@affine/graphql';
+
+import { DummyConnection } from '../../connection';
+
+export class HttpConnection extends DummyConnection {
+  readonly fetch = async (input: string, init?: RequestInit) => {
+    const externalSignal = init?.signal;
+    if (externalSignal?.aborted) {
+      throw externalSignal.reason;
+    }
+    const abortController = new AbortController();
+    externalSignal?.addEventListener('abort', reason => {
+      abortController.abort(reason);
+    });
+
+    const timeout = 15000;
+    const timeoutId = setTimeout(() => {
+      abortController.abort('timeout');
+    }, timeout);
+
+    const res = await globalThis
+      .fetch(new URL(input, this.serverBaseUrl), {
+        ...init,
+        signal: abortController.signal,
+        headers: {
+          ...init?.headers,
+          'x-affine-version': BUILD_CONFIG.appVersion,
+        },
+      })
+      .catch(err => {
+        throw new Error('fetch error: ' + err);
+      });
+    clearTimeout(timeoutId);
+    if (!res.ok && res.status !== 404) {
+      let reason: string | any = '';
+      if (res.headers.get('Content-Type')?.includes('application/json')) {
+        try {
+          reason = await res.json();
+        } catch {
+          // ignore
+        }
+      }
+      throw new Error('fetch error status: ' + res.status + ' ' + reason);
+    }
+    return res;
+  };
+
+  readonly fetchArrayBuffer = async (input: string, init?: RequestInit) => {
+    const res = await this.fetch(input, init);
+    if (res.status === 404) {
+      // 404
+      return null;
+    }
+    try {
+      return await res.arrayBuffer();
+    } catch (err) {
+      throw new Error('fetch download error: ' + err);
+    }
+  };
+
+  readonly gql = gqlFetcherFactory(
+    new URL('/graphql', this.serverBaseUrl).href,
+    this.fetch
+  );
+
+  constructor(private readonly serverBaseUrl: string) {
+    super();
+  }
+}

packages/common/nbstore/src/impls/cloud/index.ts

@@ -1,3 +1,17 @@
+import type { StorageConstructor } from '..';
+import { CloudAwarenessStorage } from './awareness';
+import { CloudBlobStorage } from './blob';
+import { CloudDocStorage } from './doc';
+import { StaticCloudDocStorage } from './doc-static';
+
 export * from './awareness';
 export * from './blob';
 export * from './doc';
+export * from './doc-static';
+
+export const cloudStorages = [
+  CloudDocStorage,
+  StaticCloudDocStorage,
+  CloudBlobStorage,
+  CloudAwarenessStorage,
+] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/cloud/socket.ts

@@ -162,7 +162,7 @@ export class SocketConnection extends AutoReconnectConnection<Socket> {
 
   constructor(
     private readonly endpoint: string,
-    private readonly socketOptions: SocketOptions
+    private readonly socketOptions?: SocketOptions
   ) {
     super();
   }