feat!: affine cloud support (#3813)

Co-authored-by: Hongtao Lye <codert.sn@gmail.com>
Co-authored-by: liuyi <forehalo@gmail.com>
Co-authored-by: LongYinan <lynweklm@gmail.com>
Co-authored-by: X1a0t <405028157@qq.com>
Co-authored-by: JimmFly <yangjinfei001@gmail.com>
Co-authored-by: Peng Xiao <pengxiao@outlook.com>
Co-authored-by: xiaodong zuo <53252747+zuoxiaodong0815@users.noreply.github.com>
Co-authored-by: DarkSky <25152247+darkskygit@users.noreply.github.com>
Co-authored-by: Qi <474021214@qq.com>
Co-authored-by: danielchim <kahungchim@gmail.com>

packages/workspace/src/providers/__tests__/socketio-provider.spec.ts

@@ -0,0 +1,103 @@
+/**
+ * @vitest-environment happy-dom
+ */
+import 'fake-indexeddb/auto';
+
+import type { AffineSocketIOProvider } from '@affine/env/workspace';
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import { Schema, Workspace } from '@blocksuite/store';
+import { describe, expect, test } from 'vitest';
+import * as awarenessProtocol from 'y-protocols/awareness';
+import { Doc } from 'yjs';
+
+import { createAffineSocketIOProvider } from '..';
+
+const schema = new Schema();
+
+schema.register(AffineSchemas).register(__unstableSchemas);
+
+describe('sockio provider', () => {
+  test.skip('test storage', async () => {
+    const workspaceId = 'test-storage-ws';
+    {
+      const workspace = new Workspace({
+        id: workspaceId,
+        isSSR: true,
+        schema,
+      });
+      const provider = createAffineSocketIOProvider(
+        workspace.id,
+        workspace.doc,
+        {
+          awareness: workspace.awarenessStore.awareness,
+        }
+      ) as AffineSocketIOProvider;
+      provider.connect();
+      const page = workspace.createPage({
+        id: 'page',
+      });
+
+      await page.waitForLoaded();
+      page.addBlock('affine:page', {
+        title: new page.Text('123123'),
+      });
+
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+
+    {
+      const workspace = new Workspace({
+        id: workspaceId,
+        isSSR: true,
+        schema,
+      });
+      const provider = createAffineSocketIOProvider(
+        workspace.id,
+        workspace.doc,
+        {
+          awareness: workspace.awarenessStore.awareness,
+        }
+      ) as AffineSocketIOProvider;
+
+      provider.connect();
+
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      const page = workspace.getPage('page')!;
+      await page.waitForLoaded();
+      const block = page.getBlockByFlavour('affine:page');
+      expect(block[0].flavour).toEqual('affine:page');
+    }
+  });
+
+  test.skip('test collaboration', async () => {
+    const workspaceId = 'test-collboration-ws';
+    {
+      const doc = new Doc({ guid: workspaceId });
+      const provider = createAffineSocketIOProvider(doc.guid, doc, {
+        awareness: new awarenessProtocol.Awareness(doc),
+      }) as AffineSocketIOProvider;
+
+      const doc2 = new Doc({ guid: workspaceId });
+      const provider2 = createAffineSocketIOProvider(doc2.guid, doc2, {
+        awareness: new awarenessProtocol.Awareness(doc2),
+      }) as AffineSocketIOProvider;
+
+      provider.connect();
+      provider2.connect();
+
+      await new Promise(resolve => setTimeout(resolve, 500));
+
+      const subdoc = new Doc();
+      const folder = doc.getMap();
+      folder.set('subDoc', subdoc);
+      subdoc.getText().insert(0, 'subDoc content');
+
+      await new Promise(resolve => setTimeout(resolve, 1000));
+
+      expect(
+        (doc2.getMap().get('subDoc') as Doc).getText().toJSON(),
+        'subDoc content'
+      );
+    }
+  });
+});

packages/workspace/src/providers/cloud/index.ts

@@ -0,0 +1,88 @@
+import { DebugLogger } from '@affine/debug';
+import { fetchWithReport } from '@affine/graphql';
+import type { ActiveDocProvider, DocProviderCreator } from '@blocksuite/store';
+import { Workspace } from '@blocksuite/store';
+import type { Doc } from 'yjs';
+
+const Y = Workspace.Y;
+
+const logger = new DebugLogger('affine:cloud');
+
+export async function downloadBinaryFromCloud(
+  rootGuid: string,
+  pageGuid: string
+) {
+  const response = await fetchWithReport(
+    runtimeConfig.serverUrlPrefix +
+      `/api/workspaces/${rootGuid}/docs/${pageGuid}`
+  );
+  if (response.ok) {
+    return response.arrayBuffer();
+  }
+  return false;
+}
+
+async function downloadBinary(rootGuid: string, doc: Doc) {
+  const buffer = await downloadBinaryFromCloud(rootGuid, doc.guid);
+  if (buffer) {
+    Y.applyUpdate(doc, new Uint8Array(buffer), 'affine-cloud');
+  }
+}
+
+export const createCloudDownloadProvider: DocProviderCreator = (
+  id,
+  doc
+): ActiveDocProvider => {
+  let _resolve: () => void;
+  let _reject: (error: unknown) => void;
+  const promise = new Promise<void>((resolve, reject) => {
+    _resolve = resolve;
+    _reject = reject;
+  });
+
+  return {
+    flavour: 'affine-cloud-download',
+    active: true,
+    sync() {
+      downloadBinary(id, doc)
+        .then(() => {
+          logger.info(`Downloaded ${id}`);
+          _resolve();
+        })
+        .catch(_reject);
+    },
+    get whenReady() {
+      return promise;
+    },
+  };
+};
+
+export const createMergeCloudSnapshotProvider: DocProviderCreator = (
+  id,
+  doc
+): ActiveDocProvider => {
+  let _resolve: () => void;
+  const promise = new Promise<void>(resolve => {
+    _resolve = resolve;
+  });
+
+  return {
+    flavour: 'affine-cloud-merge-snapshot',
+    active: true,
+    sync() {
+      downloadBinary(id, doc)
+        .then(() => {
+          logger.info(`Downloaded ${id}`);
+          _resolve();
+        })
+        // ignore error
+        .catch(e => {
+          console.error(e);
+          _resolve();
+        });
+    },
+    get whenReady() {
+      return promise;
+    },
+  };
+};

packages/workspace/src/providers/index.ts

@@ -1,8 +1,10 @@
 import { DebugLogger } from '@affine/debug';
 import type {
+  AffineSocketIOProvider,
   LocalIndexedDBBackgroundProvider,
   LocalIndexedDBDownloadProvider,
 } from '@affine/env/workspace';
+import { createLazyProvider } from '@affine/y-provider';
 import { assertExists } from '@blocksuite/global/utils';
 import type { DocProviderCreator } from '@blocksuite/store';
 import { Workspace } from '@blocksuite/store';
@@ -13,6 +15,12 @@ import {
 } from '@toeverything/y-indexeddb';
 import type { Doc } from 'yjs';
 
+import { createAffineDataSource } from '../affine';
+import {
+  createCloudDownloadProvider,
+  createMergeCloudSnapshotProvider,
+  downloadBinaryFromCloud,
+} from './cloud';
 import {
   createSQLiteDBDownloadProvider,
   createSQLiteProvider,
@@ -21,6 +29,18 @@ import {
 const Y = Workspace.Y;
 const logger = new DebugLogger('indexeddb-provider');
 
+const createAffineSocketIOProvider: DocProviderCreator = (
+  id,
+  doc,
+  { awareness }
+): AffineSocketIOProvider => {
+  const dataSource = createAffineDataSource(id, doc, awareness);
+  return {
+    flavour: 'affine-socket-io',
+    ...createLazyProvider(doc, dataSource),
+  };
+};
+
 const createIndexedDBBackgroundProvider: DocProviderCreator = (
   id,
   blockSuiteWorkspace
@@ -72,6 +92,7 @@ const createIndexedDBDownloadProvider: DocProviderCreator = (
       Y.applyUpdate(doc, binary, indexedDBDownloadOrigin);
     }
   }
+
   return {
     flavour: 'local-indexeddb',
     active: true,
@@ -89,11 +110,13 @@ const createIndexedDBDownloadProvider: DocProviderCreator = (
 };
 
 export {
+  createAffineSocketIOProvider,
   createBroadcastChannelProvider,
   createIndexedDBBackgroundProvider,
   createIndexedDBDownloadProvider,
   createSQLiteDBDownloadProvider,
   createSQLiteProvider,
+  downloadBinaryFromCloud,
 };
 
 export const createLocalProviders = (): DocProviderCreator[] => {
@@ -116,9 +139,16 @@ export const createLocalProviders = (): DocProviderCreator[] => {
 export const createAffineProviders = (): DocProviderCreator[] => {
   return (
     [
+      ...createLocalProviders(),
       runtimeConfig.enableBroadcastChannelProvider &&
         createBroadcastChannelProvider,
+      runtimeConfig.enableCloud && createAffineSocketIOProvider,
+      runtimeConfig.enableCloud && createMergeCloudSnapshotProvider,
       createIndexedDBDownloadProvider,
     ] as DocProviderCreator[]
   ).filter(v => Boolean(v));
 };
+
+export const createAffinePublicProviders = (): DocProviderCreator[] => {
+  return [createCloudDownloadProvider];
+};

packages/workspace/src/providers/sqlite-providers.ts

@@ -24,10 +24,18 @@ const createDatasource = (workspaceId: string): DatasourceDocAdapter => {
 
   return {
     queryDocState: async guid => {
-      return window.apis.db.getDocAsUpdates(
+      const update = await window.apis.db.getDocAsUpdates(
         workspaceId,
         workspaceId === guid ? undefined : guid
       );
+
+      if (update) {
+        return {
+          missing: update,
+        };
+      }
+
+      return false;
     },
     sendDocUpdate: async (guid, update) => {
       return window.apis.db.applyDocUpdate(