Merge commit 'master' into feat/cloud-sync

packages/data-center/src/datacenter.ts

@@ -46,8 +46,11 @@ export class DataCenter {
     this._providers.set(provider.id, provider);
   }
 
-  private async _getProvider(id: string, providerId: string): Promise<string> {
+  private async _getProvider(
-    const providerKey = `workspace:${id}:provider`;
+    id: string,
+    providerId = 'local'
+  ): Promise<string> {
+    const providerKey = `${id}:provider`;
     if (this._providers.has(providerId)) {
       await this._config.set(providerKey, providerId);
       return providerId;
@@ -58,21 +61,32 @@ export class DataCenter {
     throw Error(`Provider ${providerId} not found`);
   }
 
-  private async _getWorkspace(id: string, pid: string): Promise<BaseProvider> {
+  private async _getWorkspace(
-    this._logger(`Init workspace ${id} with ${pid}`);
+    id: string,
+    params: LoadConfig
+  ): Promise<BaseProvider> {
+    this._logger(`Init workspace ${id} with ${params.providerId}`);
 
-    const providerId = await this._getProvider(id, pid);
+    const providerId = await this._getProvider(id, params.providerId);
 
     // init workspace & register block schema
     const workspace = new Workspace({ room: id }).register(BlockSchema);
 
     const Provider = this._providers.get(providerId);
     assert(Provider);
-    const provider = new Provider();
 
+    // initial configurator
+    const config = getKVConfigure(`workspace:${id}`);
+    // set workspace configs
+    const values = Object.entries(params.config || {});
+    if (values.length) await config.setMany(values);
+
+    // init data by provider
+    const provider = new Provider();
     await provider.init({
       apis: this._apis,
-      config: getKVConfigure(id),
+      config,
+      globalConfig: getKVConfigure(`provider:${providerId}`),
       debug: this._logger.enabled,
       logger: this._logger.extend(`${Provider.id}:${id}`),
       workspace,
@@ -83,27 +97,22 @@ export class DataCenter {
     return provider;
   }
 
-  async setConfig(workspace: string, config: Record<string, any>) {
+  /**
-    const values = Object.entries(config);
+   * load workspace data to memory
-    if (values.length) {
+   * @param workspaceId workspace id
-      const configure = getKVConfigure(workspace);
+   * @param config.providerId provider id
-      await configure.setMany(values);
+   * @param config.config provider config
-    }
+   * @returns Workspace instance
-  }
+   */
-
-  // load workspace data to memory
   async load(
     workspaceId: string,
     params: LoadConfig = {}
   ): Promise<Workspace | null> {
-    const { providerId = 'local', config = {} } = params;
     if (workspaceId) {
       if (!this._workspaces.has(workspaceId)) {
         this._workspaces.set(
           workspaceId,
-          this.setConfig(workspaceId, config).then(() =>
+          this._getWorkspace(workspaceId, params)
-            this._getWorkspace(workspaceId, providerId)
-          )
         );
       }
       const workspace = this._workspaces.get(workspaceId);
@@ -113,7 +122,10 @@ export class DataCenter {
     return null;
   }
 
-  // destroy workspace's instance in memory
+  /**
+   * destroy workspace's instance in memory
+   * @param workspaceId workspace id
+   */
   async destroy(workspaceId: string) {
     const provider = await this._workspaces.get(workspaceId);
     if (provider) {
@@ -122,6 +134,13 @@ export class DataCenter {
     }
   }
 
+  /**
+   * reload new workspace instance to memory to refresh config
+   * @param workspaceId workspace id
+   * @param config.providerId provider id
+   * @param config.config provider config
+   * @returns Workspace instance
+   */
   async reload(
     workspaceId: string,
     config: LoadConfig = {}
@@ -130,16 +149,34 @@ export class DataCenter {
     return this.load(workspaceId, config);
   }
 
-  async list() {
+  /**
-    const keys = await this._config.keys();
+   * get workspace list
-    return keys
+   */
-      .filter(k => k.startsWith('workspace:'))
+  async list(): Promise<Record<string, Record<string, boolean>>> {
-      .map(k => k.split(':')[1]);
+    const lists = await Promise.all(
+      Array.from(this._providers.entries()).map(([providerId, provider]) =>
+        provider
+          .list(getKVConfigure(`provider:${providerId}`))
+          .then(list => [providerId, list || []] as const)
+      )
+    );
+
+    return lists.reduce((ret, [providerId, list]) => {
+      for (const [item, isLocal] of list) {
+        const workspace = ret[item] || {};
+        workspace[providerId] = isLocal;
+        ret[item] = workspace;
+      }
+      return ret;
+    }, {} as Record<string, Record<string, boolean>>);
   }
 
-  // delete local workspace's data
+  /**
+   * delete local workspace's data
+   * @param workspaceId workspace id
+   */
   async delete(workspaceId: string) {
-    await this._config.delete(`workspace:${workspaceId}:provider`);
+    await this._config.delete(`${workspaceId}:provider`);
     const provider = await this._workspaces.get(workspaceId);
     if (provider) {
       this._workspaces.delete(workspaceId);
@@ -148,9 +185,11 @@ export class DataCenter {
     }
   }
 
-  // clear all local workspace's data
+  /**
+   * clear all local workspace's data
+   */
   async clear() {
     const workspaces = await this.list();
-    await Promise.all(workspaces.map(id => this.delete(id)));
+    await Promise.all(Object.keys(workspaces).map(id => this.delete(id)));
   }
 }

packages/data-center/src/provider/affine/index.ts

@@ -51,9 +51,7 @@ export class AffineProvider extends LocalProvider {
     if (this._onTokenRefresh) {
       token.offChange(this._onTokenRefresh);
     }
-    if (this._ws) {
+    this._ws?.disconnect();
-      this._ws.disconnect();
-    }
   }
 
   async initData() {
@@ -72,9 +70,8 @@ export class AffineProvider extends LocalProvider {
             doc.once('update', resolve);
             applyUpdate(doc, new Uint8Array(updates));
           });
-          // TODO: wait util data loaded
-          this._ws = new WebsocketProvider('/', workspace.room, doc);
           // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
+          this._ws = new WebsocketProvider('/', workspace.room, doc);
           await new Promise<void>((resolve, reject) => {
             // TODO: synced will also be triggered on reconnection after losing sync
             // There needs to be an event mechanism to emit the synchronization state to the upper layer

packages/data-center/src/provider/base.ts

@@ -7,6 +7,7 @@ export class BaseProvider {
   static id = 'base';
   protected _apis!: Readonly<Apis>;
   protected _config!: Readonly<ConfigStore>;
+  protected _globalConfig!: Readonly<ConfigStore>;
   protected _logger!: Logger;
   protected _workspace!: Workspace;
 
@@ -14,9 +15,14 @@ export class BaseProvider {
     // Nothing to do here
   }
 
+  get id(): string {
+    return (this.constructor as any).id;
+  }
+
   async init(params: InitialParams) {
     this._apis = params.apis;
     this._config = params.config;
+    this._globalConfig = params.globalConfig;
     this._logger = params.logger;
     this._workspace = params.workspace;
     this._logger.enabled = params.debug;
@@ -48,4 +54,12 @@ export class BaseProvider {
   get workspace() {
     return this._workspace;
   }
+
+  // get workspace list，return a map of workspace id and boolean
+  // if value is true, it exists locally, otherwise it does not exist locally
+  static async list(
+    _config: Readonly<ConfigStore>
+  ): Promise<Map<string, boolean> | undefined> {
+    throw Error('Not implemented: list');
+  }
 }

packages/data-center/src/provider/index.ts

@@ -8,7 +8,8 @@ export type Logger = ReturnType<typeof getLogger>;
 
 export type InitialParams = {
   apis: Apis;
-  config: ConfigStore;
+  config: Readonly<ConfigStore>;
+  globalConfig: Readonly<ConfigStore>;
   debug: boolean;
   logger: Logger;
   workspace: Workspace;

packages/data-center/src/provider/local/index.ts

@@ -1,7 +1,7 @@
 import type { BlobStorage } from '@blocksuite/store';
 import assert from 'assert';
 
-import type { InitialParams } from '../index.js';
+import type { ConfigStore, InitialParams } from '../index.js';
 import { BaseProvider } from '../base.js';
 import { IndexedDBProvider } from './indexeddb.js';
 
@@ -31,19 +31,20 @@ export class LocalProvider extends BaseProvider {
 
     await this._idb.whenSynced;
     this._logger('Local data loaded');
+
+    await this._globalConfig.set(this._workspace.room, true);
   }
 
   async clear() {
     await super.clear();
     await this._blobs.clear();
-    this._idb?.clearData();
+    await this._idb?.clearData();
+    await this._globalConfig.delete(this._workspace.room!);
   }
 
   async destroy(): Promise<void> {
     super.destroy();
-    if (this._idb) {
+    await this._idb?.destroy();
-      await this._idb.destroy();
-    }
   }
 
   async getBlob(id: string): Promise<string | null> {
@@ -53,4 +54,11 @@ export class LocalProvider extends BaseProvider {
   async setBlob(blob: Blob): Promise<string> {
     return this._blobs.set(blob);
   }
+
+  static async list(
+    config: Readonly<ConfigStore<boolean>>
+  ): Promise<Map<string, boolean> | undefined> {
+    const entries = await config.entries();
+    return new Map(entries);
+  }
 }

packages/data-center/src/store.ts

@@ -1,10 +1,20 @@
-import { createStore, del, get, keys, set, setMany, clear } from 'idb-keyval';
+import {
+  createStore,
+  del,
+  get,
+  keys,
+  set,
+  setMany,
+  clear,
+  entries,
+} from 'idb-keyval';
 
 export type ConfigStore<T = any> = {
   get: (key: string) => Promise<T | undefined>;
   set: (key: string, value: T) => Promise<void>;
   setMany: (values: [string, T][]) => Promise<void>;
   keys: () => Promise<string[]>;
+  entries: () => Promise<[string, T][]>;
   delete: (key: string) => Promise<void>;
   clear: () => Promise<void>;
 };
@@ -16,6 +26,7 @@ const initialIndexedDB = <T = any>(database: string): ConfigStore<T> => {
     set: (key: string, value: T) => set(key, value, store),
     setMany: (values: [string, T][]) => setMany(values, store),
     keys: () => keys(store),
+    entries: () => entries(store),
     delete: (key: string) => del(key, store),
     clear: () => clear(store),
   };
@@ -27,9 +38,10 @@ const scopedIndexedDB = () => {
 
   return <T = any>(scope: string): Readonly<ConfigStore<T>> => {
     if (!storeCache.has(scope)) {
+      const prefix = `${scope}:`;
       const store = {
-        get: async (key: string) => idb.get(`${scope}:${key}`),
+        get: async (key: string) => idb.get(prefix + key),
-        set: (key: string, value: T) => idb.set(`${scope}:${key}`, value),
+        set: (key: string, value: T) => idb.set(prefix + key, value),
         setMany: (values: [string, T][]) =>
           idb.setMany(values.map(([k, v]) => [`${scope}:${k}`, v])),
         keys: () =>
@@ -37,16 +49,24 @@ const scopedIndexedDB = () => {
             .keys()
             .then(keys =>
               keys
-                .filter(k => k.startsWith(`${scope}:`))
+                .filter(k => k.startsWith(prefix))
-                .map(k => k.replace(`${scope}:`, ''))
+                .map(k => k.slice(prefix.length))
             ),
-        delete: (key: string) => idb.delete(`${scope}:${key}`),
+        entries: () =>
+          idb
+            .entries()
+            .then(entries =>
+              entries
+                .filter(([k]) => k.startsWith(prefix))
+                .map(([k, v]) => [k.slice(prefix.length), v] as [string, T])
+            ),
+        delete: (key: string) => idb.delete(prefix + key),
         clear: async () => {
           await idb
             .keys()
             .then(keys =>
               Promise.all(
-                keys.filter(k => k.startsWith(`${scope}:`)).map(k => del(k))
+                keys.filter(k => k.startsWith(prefix)).map(k => del(k))
               )
             );
         },

packages/data-center/tests/workspace.spec.ts

@@ -5,7 +5,17 @@ import { getDataCenter } from './utils.js';
 import 'fake-indexeddb/auto';
 
 test.describe('workspace', () => {
-  test('list workspaces', async () => {
+  test('create', async () => {});
+
+  test('load', async () => {});
+
+  test('get workspace name', async () => {});
+  test('set workspace name', async () => {});
+
+  test('get workspace avatar', async () => {});
+  test('set workspace avatar', async () => {});
+
+  test('list', async () => {
     const dataCenter = await getDataCenter();
     await dataCenter.clear();
 
@@ -16,14 +26,23 @@ test.describe('workspace', () => {
       dataCenter.load('test6'),
     ]);
 
-    expect(await dataCenter.list()).toStrictEqual([
+    expect(await dataCenter.list()).toStrictEqual({
-      'test3',
+      test3: { local: true },
-      'test4',
+      test4: { local: true },
-      'test5',
+      test5: { local: true },
-      'test6',
+      test6: { local: true },
-    ]);
+    });
+
+    await dataCenter.reload('test3', { providerId: 'affine' });
+    expect(await dataCenter.list()).toStrictEqual({
+      test3: { affine: true, local: true },
+      test4: { local: true },
+      test5: { local: true },
+      test6: { local: true },
+    });
   });
-  test('destroy workspaces', async () => {
+
+  test('destroy', async () => {
     const dataCenter = await getDataCenter();
     await dataCenter.clear();
 
@@ -39,23 +58,13 @@ test.describe('workspace', () => {
     expect(ws3 !== ws4).toBeTruthy();
   });
 
-  test('remove workspaces', async () => {
+  test('remove', async () => {
     const dataCenter = await getDataCenter();
     await dataCenter.clear();
 
     // remove workspace will remove workspace data
     await Promise.all([dataCenter.load('test9'), dataCenter.load('test10')]);
     await dataCenter.delete('test9');
-    expect(await dataCenter.list()).toStrictEqual(['test10']);
+    expect(await dataCenter.list()).toStrictEqual({ test10: { local: true } });
   });
-
-  test('create workspace', async () => {});
-  test('get the workspace', async () => {});
-
-  test('get  workspace name', async () => {});
-  test('set  workspace name', async () => {});
-
-  test('get  workspace avatar', async () => {});
-
-  test('set  workspace avatar', async () => {});
 });