refactor(infra): directory structure (#4615)

packages/frontend/workspace/src/providers/__tests__/indexeddb-provider.spec.ts

@@ -0,0 +1,86 @@
+/**
+ * @vitest-environment happy-dom
+ */
+import 'fake-indexeddb/auto';
+
+import type {
+  LocalIndexedDBBackgroundProvider,
+  LocalIndexedDBDownloadProvider,
+} from '@affine/env/workspace';
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import { Schema, Workspace } from '@blocksuite/store';
+import { afterEach, describe, expect, test } from 'vitest';
+
+import {
+  createIndexedDBBackgroundProvider,
+  createIndexedDBDownloadProvider,
+} from '..';
+
+const schema = new Schema();
+
+schema.register(AffineSchemas).register(__unstableSchemas);
+
+afterEach(() => {
+  globalThis.localStorage.clear();
+  globalThis.indexedDB.deleteDatabase('affine-local');
+});
+
+describe('download provider', () => {
+  test('basic', async () => {
+    let prev: any;
+    {
+      const workspace = new Workspace({
+        id: 'test',
+        isSSR: true,
+        schema,
+      });
+      const provider = createIndexedDBBackgroundProvider(
+        workspace.id,
+        workspace.doc,
+        {
+          awareness: workspace.awarenessStore.awareness,
+        }
+      ) as LocalIndexedDBBackgroundProvider;
+      provider.connect();
+      const page = workspace.createPage({
+        id: 'page0',
+      });
+      await page.waitForLoaded();
+      const pageBlockId = page.addBlock('affine:page', {
+        title: new page.Text(''),
+      });
+      page.addBlock('affine:surface', {}, pageBlockId);
+      const frameId = page.addBlock('affine:note', {}, pageBlockId);
+      page.addBlock('affine:paragraph', {}, frameId);
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      provider.disconnect();
+      prev = workspace.doc.toJSON();
+    }
+
+    {
+      const workspace = new Workspace({
+        id: 'test',
+        isSSR: true,
+        schema,
+      });
+      const provider = createIndexedDBDownloadProvider(
+        workspace.id,
+        workspace.doc,
+        {
+          awareness: workspace.awarenessStore.awareness,
+        }
+      ) as LocalIndexedDBDownloadProvider;
+      provider.sync();
+      await provider.whenReady;
+      expect(workspace.doc.toJSON()).toEqual({
+        ...prev,
+        // download provider only download the root doc
+        spaces: {
+          page0: {
+            blocks: {},
+          },
+        },
+      });
+    }
+  });
+});

packages/frontend/workspace/src/providers/__tests__/socketio-provider.spec.ts

@@ -0,0 +1,103 @@
+/**
+ * @vitest-environment happy-dom
+ */
+import 'fake-indexeddb/auto';
+
+import type { AffineSocketIOProvider } from '@affine/env/workspace';
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import { Schema, Workspace } from '@blocksuite/store';
+import { describe, expect, test } from 'vitest';
+import * as awarenessProtocol from 'y-protocols/awareness';
+import { Doc } from 'yjs';
+
+import { createAffineSocketIOProvider } from '..';
+
+const schema = new Schema();
+
+schema.register(AffineSchemas).register(__unstableSchemas);
+
+describe('sockio provider', () => {
+  test.skip('test storage', async () => {
+    const workspaceId = 'test-storage-ws';
+    {
+      const workspace = new Workspace({
+        id: workspaceId,
+        isSSR: true,
+        schema,
+      });
+      const provider = createAffineSocketIOProvider(
+        workspace.id,
+        workspace.doc,
+        {
+          awareness: workspace.awarenessStore.awareness,
+        }
+      ) as AffineSocketIOProvider;
+      provider.connect();
+      const page = workspace.createPage({
+        id: 'page',
+      });
+
+      await page.waitForLoaded();
+      page.addBlock('affine:page', {
+        title: new page.Text('123123'),
+      });
+
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+
+    {
+      const workspace = new Workspace({
+        id: workspaceId,
+        isSSR: true,
+        schema,
+      });
+      const provider = createAffineSocketIOProvider(
+        workspace.id,
+        workspace.doc,
+        {
+          awareness: workspace.awarenessStore.awareness,
+        }
+      ) as AffineSocketIOProvider;
+
+      provider.connect();
+
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      const page = workspace.getPage('page')!;
+      await page.waitForLoaded();
+      const block = page.getBlockByFlavour('affine:page');
+      expect(block[0].flavour).toEqual('affine:page');
+    }
+  });
+
+  test.skip('test collaboration', async () => {
+    const workspaceId = 'test-collboration-ws';
+    {
+      const doc = new Doc({ guid: workspaceId });
+      const provider = createAffineSocketIOProvider(doc.guid, doc, {
+        awareness: new awarenessProtocol.Awareness(doc),
+      }) as AffineSocketIOProvider;
+
+      const doc2 = new Doc({ guid: workspaceId });
+      const provider2 = createAffineSocketIOProvider(doc2.guid, doc2, {
+        awareness: new awarenessProtocol.Awareness(doc2),
+      }) as AffineSocketIOProvider;
+
+      provider.connect();
+      provider2.connect();
+
+      await new Promise(resolve => setTimeout(resolve, 500));
+
+      const subdoc = new Doc();
+      const folder = doc.getMap();
+      folder.set('subDoc', subdoc);
+      subdoc.getText().insert(0, 'subDoc content');
+
+      await new Promise(resolve => setTimeout(resolve, 1000));
+
+      expect(
+        (doc2.getMap().get('subDoc') as Doc).getText().toJSON(),
+        'subDoc content'
+      );
+    }
+  });
+});

packages/frontend/workspace/src/providers/__tests__/sqlite-provider.spec.ts

@@ -0,0 +1,165 @@
+import type {
+  SQLiteDBDownloadProvider,
+  SQLiteProvider,
+} from '@affine/env/workspace';
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import type { Y as YType } from '@blocksuite/store';
+import { Schema, Workspace } from '@blocksuite/store';
+import type { DBHandlerManager } from '@toeverything/infra/handler';
+import type {
+  EventMap,
+  UnwrapManagerHandlerToClientSide,
+} from '@toeverything/infra/type';
+import { nanoid } from 'nanoid';
+import { setTimeout } from 'timers/promises';
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+import { getDoc } from 'y-provider';
+
+import {
+  createSQLiteDBDownloadProvider,
+  createSQLiteProvider,
+} from '../sqlite-providers';
+
+const Y = Workspace.Y;
+
+let id: string;
+let workspace: Workspace;
+let provider: SQLiteProvider;
+let downloadProvider: SQLiteDBDownloadProvider;
+
+let offlineYdoc: YType.Doc;
+
+let triggerDBUpdate:
+  | Parameters<typeof window.events.db.onExternalUpdate>[0]
+  | null = null;
+
+const mockedAddBlob = vi.fn();
+
+vi.stubGlobal('window', {
+  apis: {
+    db: {
+      getDocAsUpdates: async (_, guid) => {
+        const subdoc = guid ? getDoc(offlineYdoc, guid) : offlineYdoc;
+        if (!subdoc) {
+          return false;
+        }
+        return Y.encodeStateAsUpdate(subdoc);
+      },
+      applyDocUpdate: async (_, update, subdocId) => {
+        const subdoc = subdocId ? getDoc(offlineYdoc, subdocId) : offlineYdoc;
+        if (!subdoc) {
+          return;
+        }
+        Y.applyUpdate(subdoc, update, 'sqlite');
+      },
+      getBlobKeys: async () => {
+        // todo: may need to hack the way to get hash keys of blobs
+        return [];
+      },
+      addBlob: mockedAddBlob,
+    } satisfies Partial<UnwrapManagerHandlerToClientSide<DBHandlerManager>>,
+  },
+  events: {
+    db: {
+      onExternalUpdate: fn => {
+        triggerDBUpdate = fn;
+        return () => {
+          triggerDBUpdate = null;
+        };
+      },
+    },
+  } as Partial<EventMap>,
+});
+
+vi.stubGlobal('environment', {
+  isDesktop: true,
+});
+
+const schema = new Schema();
+
+schema.register(AffineSchemas).register(__unstableSchemas);
+
+beforeEach(() => {
+  id = nanoid();
+  workspace = new Workspace({
+    id,
+    isSSR: true,
+    schema,
+  });
+  provider = createSQLiteProvider(workspace.id, workspace.doc, {
+    awareness: workspace.awarenessStore.awareness,
+  }) as SQLiteProvider;
+  downloadProvider = createSQLiteDBDownloadProvider(
+    workspace.id,
+    workspace.doc,
+    {
+      awareness: workspace.awarenessStore.awareness,
+    }
+  ) as SQLiteDBDownloadProvider;
+  offlineYdoc = new Y.Doc();
+  offlineYdoc.getText('text').insert(0, 'sqlite-hello');
+});
+
+describe('SQLite download provider', () => {
+  test('sync updates', async () => {
+    // on connect, the updates from sqlite should be sync'ed to the existing ydoc
+    workspace.doc.getText('text').insert(0, 'mem-hello');
+
+    downloadProvider.sync();
+    await downloadProvider.whenReady;
+
+    // depending on the nature of the sync, the data can be sync'ed in either direction
+    const options = ['sqlite-hellomem-hello', 'mem-hellosqlite-hello'];
+    const synced = options.filter(
+      o => o === workspace.doc.getText('text').toString()
+    );
+    expect(synced.length).toBe(1);
+  });
+
+  // there is no updates from sqlite for now
+  test.skip('on db update', async () => {
+    provider.connect();
+
+    await setTimeout(200);
+
+    offlineYdoc.getText('text').insert(0, 'sqlite-world');
+
+    triggerDBUpdate?.({
+      workspaceId: id + '-another-id',
+      update: Y.encodeStateAsUpdate(offlineYdoc),
+    });
+
+    // not yet updated (because the workspace id is different)
+    expect(workspace.doc.getText('text').toString()).toBe('');
+
+    triggerDBUpdate?.({
+      workspaceId: id,
+      update: Y.encodeStateAsUpdate(offlineYdoc),
+    });
+
+    expect(workspace.doc.getText('text').toString()).toBe(
+      'sqlite-worldsqlite-hello'
+    );
+  });
+
+  test('disconnect handlers', async () => {
+    const offHandler = vi.fn();
+    let handleUpdate = () => {};
+    let handleSubdocs = () => {};
+    workspace.doc.on = (event: string, fn: () => void) => {
+      if (event === 'update') {
+        handleUpdate = fn;
+      } else if (event === 'subdocs') {
+        handleSubdocs = fn;
+      }
+    };
+    workspace.doc.off = offHandler;
+    provider.connect();
+
+    provider.disconnect();
+
+    expect(triggerDBUpdate).toBe(null);
+    expect(offHandler).toBeCalledWith('update', handleUpdate);
+    expect(offHandler).toBeCalledWith('subdocs', handleSubdocs);
+  });
+});

packages/frontend/workspace/src/providers/cloud/index.ts

@@ -0,0 +1,98 @@
+import { DebugLogger } from '@affine/debug';
+import { fetchWithTraceReport } from '@affine/graphql';
+import type { ActiveDocProvider, DocProviderCreator } from '@blocksuite/store';
+import { Workspace } from '@blocksuite/store';
+import type { Doc } from 'yjs';
+
+const Y = Workspace.Y;
+
+const logger = new DebugLogger('affine:cloud');
+
+const hashMap = new Map<string, ArrayBuffer>();
+
+export async function downloadBinaryFromCloud(
+  rootGuid: string,
+  pageGuid: string
+): Promise<boolean | ArrayBuffer> {
+  if (hashMap.has(`${rootGuid}/${pageGuid}`)) {
+    return true;
+  }
+  const response = await fetchWithTraceReport(
+    runtimeConfig.serverUrlPrefix +
+      `/api/workspaces/${rootGuid}/docs/${pageGuid}`,
+    {
+      priority: 'high',
+    }
+  );
+  if (response.ok) {
+    const arrayBuffer = await response.arrayBuffer();
+    hashMap.set(`${rootGuid}/${pageGuid}`, arrayBuffer);
+    return arrayBuffer;
+  }
+  return false;
+}
+
+async function downloadBinary(rootGuid: string, doc: Doc) {
+  const buffer = await downloadBinaryFromCloud(rootGuid, doc.guid);
+  if (typeof buffer !== 'boolean') {
+    Y.applyUpdate(doc, new Uint8Array(buffer), 'affine-cloud');
+  }
+}
+
+export const createCloudDownloadProvider: DocProviderCreator = (
+  id,
+  doc
+): ActiveDocProvider => {
+  let _resolve: () => void;
+  let _reject: (error: unknown) => void;
+  const promise = new Promise<void>((resolve, reject) => {
+    _resolve = resolve;
+    _reject = reject;
+  });
+
+  return {
+    flavour: 'affine-cloud-download',
+    active: true,
+    sync() {
+      downloadBinary(id, doc)
+        .then(() => {
+          logger.info(`Downloaded ${id}`);
+          _resolve();
+        })
+        .catch(_reject);
+    },
+    get whenReady() {
+      return promise;
+    },
+  };
+};
+
+export const createMergeCloudSnapshotProvider: DocProviderCreator = (
+  id,
+  doc
+): ActiveDocProvider => {
+  let _resolve: () => void;
+  const promise = new Promise<void>(resolve => {
+    _resolve = resolve;
+  });
+
+  return {
+    flavour: 'affine-cloud-merge-snapshot',
+    active: true,
+    sync() {
+      downloadBinary(id, doc)
+        .then(() => {
+          logger.info(`Downloaded ${id}`);
+          _resolve();
+        })
+        // ignore error
+        .catch(e => {
+          console.error(e);
+          _resolve();
+        });
+    },
+    get whenReady() {
+      return promise;
+    },
+  };
+};

packages/frontend/workspace/src/providers/index.ts

@@ -0,0 +1,163 @@
+import { DebugLogger } from '@affine/debug';
+import type {
+  AffineSocketIOProvider,
+  LocalIndexedDBBackgroundProvider,
+  LocalIndexedDBDownloadProvider,
+} from '@affine/env/workspace';
+import { assertExists } from '@blocksuite/global/utils';
+import type { DocProviderCreator } from '@blocksuite/store';
+import { Workspace } from '@blocksuite/store';
+import { createBroadcastChannelProvider } from '@blocksuite/store/providers/broadcast-channel';
+import {
+  createIndexedDBDatasource,
+  createIndexedDBProvider as create,
+} from '@toeverything/y-indexeddb';
+import { createLazyProvider } from 'y-provider';
+import { encodeStateVector } from 'yjs';
+
+import { createAffineDataSource } from '../affine';
+import {
+  createCloudDownloadProvider,
+  createMergeCloudSnapshotProvider,
+  downloadBinaryFromCloud,
+} from './cloud';
+import {
+  createSQLiteDBDownloadProvider,
+  createSQLiteProvider,
+} from './sqlite-providers';
+
+const Y = Workspace.Y;
+const logger = new DebugLogger('indexeddb-provider');
+
+const createAffineSocketIOProvider: DocProviderCreator = (
+  id,
+  doc,
+  { awareness }
+): AffineSocketIOProvider => {
+  const dataSource = createAffineDataSource(id, doc, awareness);
+  const lazyProvider = createLazyProvider(doc, dataSource, {
+    origin: 'affine-socket-io',
+  });
+  return {
+    flavour: 'affine-socket-io',
+    ...lazyProvider,
+    get status() {
+      return lazyProvider.status;
+    },
+  };
+};
+
+const createIndexedDBBackgroundProvider: DocProviderCreator = (
+  id,
+  blockSuiteWorkspace
+): LocalIndexedDBBackgroundProvider => {
+  const indexeddbProvider = create(blockSuiteWorkspace);
+
+  let connected = false;
+  return {
+    flavour: 'local-indexeddb-background',
+    datasource: indexeddbProvider.datasource,
+    passive: true,
+    get status() {
+      return indexeddbProvider.status;
+    },
+    subscribeStatusChange: indexeddbProvider.subscribeStatusChange,
+    get connected() {
+      return connected;
+    },
+    cleanup: () => {
+      indexeddbProvider.cleanup().catch(console.error);
+    },
+    connect: () => {
+      logger.info('connect indexeddb provider', id);
+      indexeddbProvider.connect();
+    },
+    disconnect: () => {
+      assertExists(indexeddbProvider);
+      logger.info('disconnect indexeddb provider', id);
+      indexeddbProvider.disconnect();
+      connected = false;
+    },
+  };
+};
+
+const indexedDBDownloadOrigin = 'indexeddb-download-provider';
+
+const createIndexedDBDownloadProvider: DocProviderCreator = (
+  id,
+  doc
+): LocalIndexedDBDownloadProvider => {
+  const datasource = createIndexedDBDatasource({});
+  let _resolve: () => void;
+  let _reject: (error: unknown) => void;
+  const promise = new Promise<void>((resolve, reject) => {
+    _resolve = resolve;
+    _reject = reject;
+  });
+
+  return {
+    flavour: 'local-indexeddb',
+    active: true,
+    get whenReady() {
+      return promise;
+    },
+    cleanup: () => {
+      // todo: cleanup data
+    },
+    sync: () => {
+      logger.info('sync indexeddb provider', id);
+      datasource
+        .queryDocState(doc.guid, {
+          stateVector: encodeStateVector(doc),
+        })
+        .then(docState => {
+          if (docState) {
+            Y.applyUpdate(doc, docState.missing, indexedDBDownloadOrigin);
+          }
+          _resolve();
+        })
+        .catch(_reject);
+    },
+  };
+};
+
+export {
+  createAffineSocketIOProvider,
+  createBroadcastChannelProvider,
+  createIndexedDBBackgroundProvider,
+  createIndexedDBDownloadProvider,
+  createSQLiteDBDownloadProvider,
+  createSQLiteProvider,
+  downloadBinaryFromCloud,
+};
+
+export const createLocalProviders = (): DocProviderCreator[] => {
+  const providers = [
+    createIndexedDBBackgroundProvider,
+    createIndexedDBDownloadProvider,
+  ] as DocProviderCreator[];
+
+  if (runtimeConfig.enableBroadcastChannelProvider) {
+    providers.push(createBroadcastChannelProvider);
+  }
+
+  if (environment.isDesktop && runtimeConfig.enableSQLiteProvider) {
+    providers.push(createSQLiteProvider, createSQLiteDBDownloadProvider);
+  }
+
+  return providers;
+};
+
+export const createAffineProviders = (): DocProviderCreator[] => {
+  return (
+    [
+      ...createLocalProviders(),
+      runtimeConfig.enableCloud && createAffineSocketIOProvider,
+      runtimeConfig.enableCloud && createMergeCloudSnapshotProvider,
+    ] as DocProviderCreator[]
+  ).filter(v => Boolean(v));
+};
+
+export const createAffinePublicProviders = (): DocProviderCreator[] => {
+  return [createCloudDownloadProvider];
+};

packages/frontend/workspace/src/providers/logger.ts

@@ -0,0 +1,3 @@
+import { DebugLogger } from '@affine/debug';
+
+export const localProviderLogger = new DebugLogger('local-provider');

packages/frontend/workspace/src/providers/sqlite-providers.ts

@@ -0,0 +1,133 @@
+import type {
+  SQLiteDBDownloadProvider,
+  SQLiteProvider,
+} from '@affine/env/workspace';
+import { assertExists } from '@blocksuite/global/utils';
+import type { DocProviderCreator } from '@blocksuite/store';
+import { Workspace as BlockSuiteWorkspace } from '@blocksuite/store';
+import { createLazyProvider, type DocDataSource } from 'y-provider';
+import type { Doc } from 'yjs';
+
+import { localProviderLogger as logger } from './logger';
+
+const Y = BlockSuiteWorkspace.Y;
+
+const sqliteOrigin = 'sqlite-provider-origin';
+
+const createDatasource = (workspaceId: string): DocDataSource => {
+  if (!window.apis?.db) {
+    throw new Error('sqlite datasource is not available');
+  }
+
+  return {
+    queryDocState: async guid => {
+      const update = await window.apis.db.getDocAsUpdates(
+        workspaceId,
+        workspaceId === guid ? undefined : guid
+      );
+
+      if (update) {
+        return {
+          missing: update,
+        };
+      }
+
+      return false;
+    },
+    sendDocUpdate: async (guid, update) => {
+      return window.apis.db.applyDocUpdate(
+        workspaceId,
+        update,
+        workspaceId === guid ? undefined : guid
+      );
+    },
+  };
+};
+
+/**
+ * A provider that is responsible for syncing updates the workspace with the local SQLite database.
+ */
+export const createSQLiteProvider: DocProviderCreator = (
+  id,
+  rootDoc
+): SQLiteProvider => {
+  const datasource = createDatasource(id);
+  let provider: ReturnType<typeof createLazyProvider> | null = null;
+  let connected = false;
+  return {
+    flavour: 'sqlite',
+    datasource,
+    passive: true,
+    get status() {
+      assertExists(provider);
+      return provider.status;
+    },
+    subscribeStatusChange(onStatusChange) {
+      assertExists(provider);
+      return provider.subscribeStatusChange(onStatusChange);
+    },
+    connect: () => {
+      provider = createLazyProvider(rootDoc, datasource, { origin: 'sqlite' });
+      provider.connect();
+      connected = true;
+    },
+    disconnect: () => {
+      provider?.disconnect();
+      provider = null;
+      connected = false;
+    },
+    get connected() {
+      return connected;
+    },
+  };
+};
+
+/**
+ * A provider that is responsible for DOWNLOADING updates from the local SQLite database.
+ */
+export const createSQLiteDBDownloadProvider: DocProviderCreator = (
+  id,
+  rootDoc
+): SQLiteDBDownloadProvider => {
+  const { apis } = window;
+
+  let _resolve: () => void;
+  let _reject: (error: unknown) => void;
+  const promise = new Promise<void>((resolve, reject) => {
+    _resolve = resolve;
+    _reject = reject;
+  });
+
+  async function syncUpdates(doc: Doc) {
+    logger.info('syncing updates from sqlite', doc.guid);
+    const subdocId = doc.guid === id ? undefined : doc.guid;
+    const updates = await apis.db.getDocAsUpdates(id, subdocId);
+
+    if (updates) {
+      Y.applyUpdate(doc, updates, sqliteOrigin);
+    }
+
+    return true;
+  }
+
+  return {
+    flavour: 'sqlite-download',
+    active: true,
+    get whenReady() {
+      return promise;
+    },
+    cleanup: () => {
+      // todo
+    },
+    sync: () => {
+      logger.info('connect sqlite download provider', id);
+      syncUpdates(rootDoc)
+        .then(() => {
+          _resolve();
+        })
+        .catch(error => {
+          _reject(error);
+        });
+    },
+  };
+};