Merge remote-tracking branch 'refs/remotes/origin/feat/cloud-sync'

Conflicts:
	packages/data-center/src/provider/base.ts

packages/data-center/src/apis/token.ts

@@ -60,7 +60,9 @@ class Token {
   }
 
   async initToken(token: string) {
-    this._setToken(await login({ token, type: 'Google' }));
+    const tokens = await login({ token, type: 'Google' });
+    this._setToken(tokens);
+    return this._user;
   }
 
   async refreshToken(token?: string) {
@@ -153,10 +155,27 @@ export const getAuthorizer = () => {
 
     const googleAuthProvider = new GoogleAuthProvider();
 
+    const getToken = async () => {
+      const currentUser = firebaseAuth.currentUser;
+      if (currentUser) {
+        await currentUser.getIdTokenResult(true);
+        if (!currentUser.isAnonymous) {
+          return currentUser.getIdToken();
+        }
+      }
+      return;
+    };
+
     const signInWithGoogle = async () => {
-      const user = await signInWithPopup(firebaseAuth, googleAuthProvider);
-      const idToken = await user.user.getIdToken();
-      await token.initToken(idToken);
+      const idToken = await getToken();
+      if (idToken) {
+        await token.initToken(idToken);
+      } else {
+        const user = await signInWithPopup(firebaseAuth, googleAuthProvider);
+        const idToken = await user.user.getIdToken();
+        await token.initToken(idToken);
+      }
+      return firebaseAuth.currentUser;
     };
 
     const onAuthStateChanged = (callback: (user: User | null) => void) => {

packages/data-center/src/datacenter.ts

@@ -1,6 +1,6 @@
 import assert from 'assert';
 import { BlockSchema } from '@blocksuite/blocks/models';
-import { Workspace } from '@blocksuite/store';
+import { Workspace, Signal } from '@blocksuite/store';
 
 import { getLogger } from './index.js';
 import { getApis, Apis } from './apis/index.js';
@@ -16,6 +16,17 @@ type LoadConfig = {
   config?: Record<string, any>;
 };
 
+export type DataCenterSignals = DataCenter['signals'];
+type WorkspaceItem = {
+  // provider id
+  provider: string;
+  // data exists locally
+  locally: boolean;
+};
+type WorkspaceLoadEvent = WorkspaceItem & {
+  workspace: string;
+};
+
 export class DataCenter {
   private readonly _apis: Apis;
   private readonly _providers = new Map<string, typeof BaseProvider>();
@@ -23,6 +34,11 @@ export class DataCenter {
   private readonly _config;
   private readonly _logger;
 
+  readonly signals = {
+    listAdd: new Signal<WorkspaceLoadEvent>(),
+    listRemove: new Signal<string>(),
+  };
+
   static async init(debug: boolean): Promise<DataCenter> {
     const dc = new DataCenter(debug);
     dc.addProvider(AffineProvider);
@@ -36,6 +52,16 @@ export class DataCenter {
     this._config = getKVConfigure('sys');
     this._logger = getLogger('dc');
     this._logger.enabled = debug;
+
+    this.signals.listAdd.on(e => {
+      this._config.set(`list:${e.workspace}`, {
+        provider: e.provider,
+        locally: e.locally,
+      });
+    });
+    this.signals.listRemove.on(workspace => {
+      this._config.delete(`list:${workspace}`);
+    });
   }
 
   get apis(): Readonly<Apis> {
@@ -86,9 +112,9 @@ export class DataCenter {
     await provider.init({
       apis: this._apis,
       config,
-      globalConfig: getKVConfigure(`provider:${providerId}`),
       debug: this._logger.enabled,
       logger: this._logger.extend(`${Provider.id}:${id}`),
+      signals: this.signals,
       workspace,
     });
     await provider.initData();
@@ -97,6 +123,21 @@ export class DataCenter {
     return provider;
   }
 
+  async auth(providerId: string, globalConfig?: Record<string, any>) {
+    const Provider = this._providers.get(providerId);
+    if (Provider) {
+      // initial configurator
+      const config = getKVConfigure(`provider:${providerId}`);
+      // set workspace configs
+      const values = Object.entries(globalConfig || {});
+      if (values.length) await config.setMany(values);
+
+      const logger = this._logger.extend(`auth:${providerId}`);
+      logger.enabled = this._logger.enabled;
+      await Provider.auth(config, logger, this.signals);
+    }
+  }
+
   /**
    * load workspace data to memory
    * @param workspaceId workspace id
@@ -150,24 +191,18 @@ export class DataCenter {
   }
 
   /**
-   * get workspace list
+   * get workspace list，return a map of workspace id and data state
+   * data state is also map, the key is the provider id, and the data exists locally when the value is true, otherwise it does not exist
    */
   async list(): Promise<Record<string, Record<string, boolean>>> {
-    const lists = await Promise.all(
-      Array.from(this._providers.entries()).map(([providerId, provider]) =>
-        provider
-          .list(getKVConfigure(`provider:${providerId}`))
-          .then(list => [providerId, list || []] as const)
-      )
-    );
-
-    return lists.reduce((ret, [providerId, list]) => {
-      for (const [item, isLocal] of list) {
-        const workspace = ret[item] || {};
-        workspace[providerId] = isLocal;
-        ret[item] = workspace;
+    const entries: [string, WorkspaceItem][] = await this._config.entries();
+    return entries.reduce((acc, [k, i]) => {
+      if (k.startsWith('list:')) {
+        const key = k.slice(5);
+        acc[key] = acc[key] || {};
+        acc[key][i.provider] = i.locally;
       }
-      return ret;
+      return acc;
     }, {} as Record<string, Record<string, boolean>>);
   }
 

packages/data-center/src/index.ts

@@ -7,6 +7,17 @@ const _initializeDataCenter = () => {
   return (debug = true) => {
     if (!_dataCenterInstance) {
       _dataCenterInstance = DataCenter.init(debug);
+      _dataCenterInstance.then(dc => {
+        try {
+          if (window) {
+            (window as any).dc = dc;
+          }
+        } catch (_) {
+          // ignore
+        }
+
+        return dc;
+      });
     }
 
     return _dataCenterInstance;

packages/data-center/src/provider/affine/index.ts

@@ -1,11 +1,17 @@
 import assert from 'assert';
-import { applyUpdate } from 'yjs';
+import { applyUpdate, Doc } from 'yjs';
 
-import type { InitialParams } from '../index.js';
-import { token, Callback } from '../../apis/index.js';
+import type {
+  ConfigStore,
+  DataCenterSignals,
+  InitialParams,
+  Logger,
+} from '../index.js';
+import { token, Callback, getApis } from '../../apis/index.js';
 import { LocalProvider } from '../local/index.js';
 
 import { WebsocketProvider } from './sync.js';
+import { IndexedDBProvider } from '../local/indexeddb.js';
 
 export class AffineProvider extends LocalProvider {
   static id = 'affine';
@@ -55,7 +61,14 @@ export class AffineProvider extends LocalProvider {
   }
 
   async initData() {
-    await super.initData();
+    const databases = await indexedDB.databases();
+    await super.initData(
+      // set locally to true if exists a same name db
+      databases
+        .map(db => db.name)
+        .filter(v => v)
+        .includes(this._workspace.room)
+    );
 
     const workspace = this._workspace;
     const doc = workspace.doc;
@@ -64,23 +77,29 @@ export class AffineProvider extends LocalProvider {
 
     if (workspace.room && token.isLogin) {
       try {
-        const updates = await this._apis.downloadWorkspace(workspace.room);
-        if (updates) {
-          await new Promise(resolve => {
-            doc.once('update', resolve);
-            applyUpdate(doc, new Uint8Array(updates));
-          });
-          // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
-          this._ws = new WebsocketProvider('/', workspace.room, doc);
-          await new Promise<void>((resolve, reject) => {
-            // TODO: synced will also be triggered on reconnection after losing sync
-            // There needs to be an event mechanism to emit the synchronization state to the upper layer
-            assert(this._ws);
-            this._ws.once('synced', () => resolve());
-            this._ws.once('lost-connection', () => resolve());
-            this._ws.once('connection-error', () => reject());
-          });
-        }
+        // init data from cloud
+        await AffineProvider._initCloudDoc(
+          workspace.room,
+          doc,
+          this._logger,
+          this._signals
+        );
+
+        // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
+        this._ws = new WebsocketProvider('/', workspace.room, doc);
+        await new Promise<void>((resolve, reject) => {
+          // TODO: synced will also be triggered on reconnection after losing sync
+          // There needs to be an event mechanism to emit the synchronization state to the upper layer
+          assert(this._ws);
+          this._ws.once('synced', () => resolve());
+          this._ws.once('lost-connection', () => resolve());
+          this._ws.once('connection-error', () => reject());
+        });
+        this._signals.listAdd.emit({
+          workspace: workspace.room,
+          provider: this.id,
+          locally: true,
+        });
       } catch (e) {
         this._logger('Failed to init cloud workspace', e);
       }
@@ -91,4 +110,66 @@ export class AffineProvider extends LocalProvider {
     // just a workaround for yjs
     doc.getMap('space:meta');
   }
+
+  private static async _initCloudDoc(
+    workspace: string,
+    doc: Doc,
+    logger: Logger,
+    signals: DataCenterSignals
+  ) {
+    const apis = getApis();
+    logger(`Loading ${workspace}...`);
+    const updates = await apis.downloadWorkspace(workspace);
+    if (updates) {
+      await new Promise(resolve => {
+        doc.once('update', resolve);
+        applyUpdate(doc, new Uint8Array(updates));
+      });
+      logger(`Loaded: ${workspace}`);
+
+      // only add to list as online workspace
+      signals.listAdd.emit({
+        workspace,
+        provider: this.id,
+        // at this time we always download full workspace
+        // but after we support sub doc, we can only download metadata
+        locally: false,
+      });
+    }
+  }
+
+  static async auth(
+    config: Readonly<ConfigStore<string>>,
+    logger: Logger,
+    signals: DataCenterSignals
+  ) {
+    const refreshToken = await config.get('token');
+    if (refreshToken) {
+      await token.refreshToken(refreshToken);
+      if (token.isLogin && !token.isExpired) {
+        logger('check login success');
+        // login success
+        return;
+      }
+    }
+
+    logger('start login');
+    // login with google
+    const apis = getApis();
+    assert(apis.signInWithGoogle);
+    const user = await apis.signInWithGoogle();
+    assert(user);
+    logger(`login success: ${user.displayName}`);
+
+    // TODO: refresh local workspace data
+    const workspaces = await apis.getWorkspaces();
+    await Promise.all(
+      workspaces.map(async ({ id }) => {
+        const doc = new Doc();
+        const idb = new IndexedDBProvider(id, doc);
+        await idb.whenSynced;
+        await this._initCloudDoc(id, doc, logger, signals);
+      })
+    );
+  }
 }

packages/data-center/src/provider/base.ts

@@ -1,15 +1,21 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
 import type { Workspace } from '@blocksuite/store';
 
-import type { Apis, Logger, InitialParams, ConfigStore } from './index';
 import type { BlobURL } from '@blocksuite/store/dist/blob/types';
+import type {
+  Apis,
+  DataCenterSignals,
+  Logger,
+  InitialParams,
+  ConfigStore,
+} from './index';
 
 export class BaseProvider {
   static id = 'base';
   protected _apis!: Readonly<Apis>;
   protected _config!: Readonly<ConfigStore>;
-  protected _globalConfig!: Readonly<ConfigStore>;
   protected _logger!: Logger;
+  protected _signals!: DataCenterSignals;
   protected _workspace!: Workspace;
 
   constructor() {
@@ -23,8 +29,8 @@ export class BaseProvider {
   async init(params: InitialParams) {
     this._apis = params.apis;
     this._config = params.config;
-    this._globalConfig = params.globalConfig;
     this._logger = params.logger;
+    this._signals = params.signals;
     this._workspace = params.workspace;
     this._logger.enabled = params.debug;
   }
@@ -58,6 +64,14 @@ export class BaseProvider {
     return this._workspace;
   }
 
+  static async auth(
+    _config: Readonly<ConfigStore>,
+    logger: Logger,
+    _signals: DataCenterSignals
+  ) {
+    logger("This provider doesn't require authentication");
+  }
+
   // get workspace list，return a map of workspace id and boolean
   // if value is true, it exists locally, otherwise it does not exist locally
   static async list(

packages/data-center/src/provider/index.ts

@@ -1,6 +1,7 @@
 import type { Workspace } from '@blocksuite/store';
 
 import type { Apis } from '../apis';
+import type { DataCenterSignals } from '../datacenter';
 import type { getLogger } from '../index';
 import type { ConfigStore } from '../store';
 
@@ -9,13 +10,13 @@ export type Logger = ReturnType<typeof getLogger>;
 export type InitialParams = {
   apis: Apis;
   config: Readonly<ConfigStore>;
-  globalConfig: Readonly<ConfigStore>;
   debug: boolean;
   logger: Logger;
+  signals: DataCenterSignals;
   workspace: Workspace;
 };
 
-export type { Apis, ConfigStore, Workspace };
+export type { Apis, ConfigStore, DataCenterSignals, Workspace };
 export type { BaseProvider } from './base.js';
 export { AffineProvider } from './affine/index.js';
 export { LocalProvider } from './local/index.js';

packages/data-center/src/provider/local/index.ts

@@ -21,7 +21,7 @@ export class LocalProvider extends BaseProvider {
     this._blobs = blobs;
   }
 
-  async initData() {
+  async initData(locally = true) {
     assert(this._workspace.room);
     this._logger('Loading local data');
     this._idb = new IndexedDBProvider(
@@ -32,14 +32,19 @@ export class LocalProvider extends BaseProvider {
     await this._idb.whenSynced;
     this._logger('Local data loaded');
 
-    await this._globalConfig.set(this._workspace.room, true);
+    this._signals.listAdd.emit({
+      workspace: this._workspace.room,
+      provider: this.id,
+      locally,
+    });
   }
 
   async clear() {
+    assert(this._workspace.room);
     await super.clear();
     await this._blobs.clear();
     await this._idb?.clearData();
-    await this._globalConfig.delete(this._workspace.room!);
+    this._signals.listRemove.emit(this._workspace.room);
   }
 
   async destroy(): Promise<void> {
@@ -59,6 +64,10 @@ export class LocalProvider extends BaseProvider {
     config: Readonly<ConfigStore<boolean>>
   ): Promise<Map<string, boolean> | undefined> {
     const entries = await config.entries();
-    return new Map(entries);
+    return new Map(
+      entries
+        .filter(([key]) => key.startsWith('list:'))
+        .map(([key, value]) => [key.slice(5), value])
+    );
   }
 }