Merge branch 'origin/feat/cloud-sync' into 'feat/cloud-sync'

packages/data-center/src/datacenter.ts

@@ -46,8 +46,11 @@ export class DataCenter {
     this._providers.set(provider.id, provider);
   }
 
-  private async _getProvider(id: string, providerId: string): Promise<string> {
-    const providerKey = `workspace:${id}:provider`;
+  private async _getProvider(
+    id: string,
+    providerId = 'local'
+  ): Promise<string> {
+    const providerKey = `${id}:provider`;
     if (this._providers.has(providerId)) {
       await this._config.set(providerKey, providerId);
       return providerId;
@@ -58,21 +61,32 @@ export class DataCenter {
     throw Error(`Provider ${providerId} not found`);
   }
 
-  private async _getWorkspace(id: string, pid: string): Promise<BaseProvider> {
-    this._logger(`Init workspace ${id} with ${pid}`);
+  private async _getWorkspace(
+    id: string,
+    params: LoadConfig
+  ): Promise<BaseProvider> {
+    this._logger(`Init workspace ${id} with ${params.providerId}`);
 
-    const providerId = await this._getProvider(id, pid);
+    const providerId = await this._getProvider(id, params.providerId);
 
     // init workspace & register block schema
     const workspace = new Workspace({ room: id }).register(BlockSchema);
 
     const Provider = this._providers.get(providerId);
     assert(Provider);
-    const provider = new Provider();
 
+    // initial configurator
+    const config = getKVConfigure(`workspace:${id}`);
+    // set workspace configs
+    const values = Object.entries(params.config || {});
+    if (values.length) await config.setMany(values);
+
+    // init data by provider
+    const provider = new Provider();
     await provider.init({
       apis: this._apis,
-      config: getKVConfigure(id),
+      config,
+      globalConfig: getKVConfigure(`provider:${providerId}`),
       debug: this._logger.enabled,
       logger: this._logger.extend(`${Provider.id}:${id}`),
       workspace,
@@ -83,27 +97,22 @@ export class DataCenter {
     return provider;
   }
 
-  async setConfig(workspace: string, config: Record<string, any>) {
-    const values = Object.entries(config);
-    if (values.length) {
-      const configure = getKVConfigure(workspace);
-      await configure.setMany(values);
-    }
-  }
-
-  // load workspace data to memory
+  /**
+   * load workspace data to memory
+   * @param workspaceId workspace id
+   * @param config.providerId provider id
+   * @param config.config provider config
+   * @returns Workspace instance
+   */
   async load(
     workspaceId: string,
     params: LoadConfig = {}
   ): Promise<Workspace | null> {
-    const { providerId = 'local', config = {} } = params;
     if (workspaceId) {
       if (!this._workspaces.has(workspaceId)) {
         this._workspaces.set(
           workspaceId,
-          this.setConfig(workspaceId, config).then(() =>
-            this._getWorkspace(workspaceId, providerId)
-          )
+          this._getWorkspace(workspaceId, params)
         );
       }
       const workspace = this._workspaces.get(workspaceId);
@@ -113,7 +122,10 @@ export class DataCenter {
     return null;
   }
 
-  // destroy workspace's instance in memory
+  /**
+   * destroy workspace's instance in memory
+   * @param workspaceId workspace id
+   */
   async destroy(workspaceId: string) {
     const provider = await this._workspaces.get(workspaceId);
     if (provider) {
@@ -122,6 +134,13 @@ export class DataCenter {
     }
   }
 
+  /**
+   * reload new workspace instance to memory to refresh config
+   * @param workspaceId workspace id
+   * @param config.providerId provider id
+   * @param config.config provider config
+   * @returns Workspace instance
+   */
   async reload(
     workspaceId: string,
     config: LoadConfig = {}
@@ -130,16 +149,34 @@ export class DataCenter {
     return this.load(workspaceId, config);
   }
 
-  async list() {
-    const keys = await this._config.keys();
-    return keys
-      .filter(k => k.startsWith('workspace:'))
-      .map(k => k.split(':')[1]);
+  /**
+   * get workspace list
+   */
+  async list(): Promise<Record<string, Record<string, boolean>>> {
+    const lists = await Promise.all(
+      Array.from(this._providers.entries()).map(([providerId, provider]) =>
+        provider
+          .list(getKVConfigure(`provider:${providerId}`))
+          .then(list => [providerId, list || []] as const)
+      )
+    );
+
+    return lists.reduce((ret, [providerId, list]) => {
+      for (const [item, isLocal] of list) {
+        const workspace = ret[item] || {};
+        workspace[providerId] = isLocal;
+        ret[item] = workspace;
+      }
+      return ret;
+    }, {} as Record<string, Record<string, boolean>>);
   }
 
-  // delete local workspace's data
+  /**
+   * delete local workspace's data
+   * @param workspaceId workspace id
+   */
   async delete(workspaceId: string) {
-    await this._config.delete(`workspace:${workspaceId}:provider`);
+    await this._config.delete(`${workspaceId}:provider`);
     const provider = await this._workspaces.get(workspaceId);
     if (provider) {
       this._workspaces.delete(workspaceId);
@@ -148,9 +185,11 @@ export class DataCenter {
     }
   }
 
-  // clear all local workspace's data
+  /**
+   * clear all local workspace's data
+   */
   async clear() {
     const workspaces = await this.list();
-    await Promise.all(workspaces.map(id => this.delete(id)));
+    await Promise.all(Object.keys(workspaces).map(id => this.delete(id)));
   }
 }

packages/data-center/src/provider/affine/index.ts

@@ -51,9 +51,7 @@ export class AffineProvider extends LocalProvider {
     if (this._onTokenRefresh) {
       token.offChange(this._onTokenRefresh);
     }
-    if (this._ws) {
-      this._ws.disconnect();
-    }
+    this._ws?.disconnect();
   }
 
   async initData() {
@@ -72,9 +70,8 @@ export class AffineProvider extends LocalProvider {
             doc.once('update', resolve);
             applyUpdate(doc, new Uint8Array(updates));
           });
-          // TODO: wait util data loaded
-          this._ws = new WebsocketProvider('/', workspace.room, doc);
           // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
+          this._ws = new WebsocketProvider('/', workspace.room, doc);
           await new Promise<void>((resolve, reject) => {
             // TODO: synced will also be triggered on reconnection after losing sync
             // There needs to be an event mechanism to emit the synchronization state to the upper layer

packages/data-center/src/provider/base.ts

@@ -7,6 +7,7 @@ export class BaseProvider {
   static id = 'base';
   protected _apis!: Readonly<Apis>;
   protected _config!: Readonly<ConfigStore>;
+  protected _globalConfig!: Readonly<ConfigStore>;
   protected _logger!: Logger;
   protected _workspace!: Workspace;
 
@@ -14,9 +15,14 @@ export class BaseProvider {
     // Nothing to do here
   }
 
+  get id(): string {
+    return (this.constructor as any).id;
+  }
+
   async init(params: InitialParams) {
     this._apis = params.apis;
     this._config = params.config;
+    this._globalConfig = params.globalConfig;
     this._logger = params.logger;
     this._workspace = params.workspace;
     this._logger.enabled = params.debug;
@@ -48,4 +54,12 @@ export class BaseProvider {
   get workspace() {
     return this._workspace;
   }
+
+  // get workspace list，return a map of workspace id and boolean
+  // if value is true, it exists locally, otherwise it does not exist locally
+  static async list(
+    _config: Readonly<ConfigStore>
+  ): Promise<Map<string, boolean> | undefined> {
+    throw Error('Not implemented: list');
+  }
 }

packages/data-center/src/provider/index.ts

@@ -8,7 +8,8 @@ export type Logger = ReturnType<typeof getLogger>;
 
 export type InitialParams = {
   apis: Apis;
-  config: ConfigStore;
+  config: Readonly<ConfigStore>;
+  globalConfig: Readonly<ConfigStore>;
   debug: boolean;
   logger: Logger;
   workspace: Workspace;

packages/data-center/src/provider/local/index.ts

@@ -1,7 +1,7 @@
 import type { BlobStorage } from '@blocksuite/store';
 import assert from 'assert';
 
-import type { InitialParams } from '../index.js';
+import type { ConfigStore, InitialParams } from '../index.js';
 import { BaseProvider } from '../base.js';
 import { IndexedDBProvider } from './indexeddb.js';
 
@@ -31,19 +31,20 @@ export class LocalProvider extends BaseProvider {
 
     await this._idb.whenSynced;
     this._logger('Local data loaded');
+
+    await this._globalConfig.set(this._workspace.room, true);
   }
 
   async clear() {
     await super.clear();
     await this._blobs.clear();
-    this._idb?.clearData();
+    await this._idb?.clearData();
+    await this._globalConfig.delete(this._workspace.room!);
   }
 
   async destroy(): Promise<void> {
     super.destroy();
-    if (this._idb) {
-      await this._idb.destroy();
-    }
+    await this._idb?.destroy();
   }
 
   async getBlob(id: string): Promise<string | null> {
@@ -53,4 +54,11 @@ export class LocalProvider extends BaseProvider {
   async setBlob(blob: Blob): Promise<string> {
     return this._blobs.set(blob);
   }
+
+  static async list(
+    config: Readonly<ConfigStore<boolean>>
+  ): Promise<Map<string, boolean> | undefined> {
+    const entries = await config.entries();
+    return new Map(entries);
+  }
 }

packages/data-center/src/store.ts

@@ -1,10 +1,20 @@
-import { createStore, del, get, keys, set, setMany, clear } from 'idb-keyval';
+import {
+  createStore,
+  del,
+  get,
+  keys,
+  set,
+  setMany,
+  clear,
+  entries,
+} from 'idb-keyval';
 
 export type ConfigStore<T = any> = {
   get: (key: string) => Promise<T | undefined>;
   set: (key: string, value: T) => Promise<void>;
   setMany: (values: [string, T][]) => Promise<void>;
   keys: () => Promise<string[]>;
+  entries: () => Promise<[string, T][]>;
   delete: (key: string) => Promise<void>;
   clear: () => Promise<void>;
 };
@@ -16,6 +26,7 @@ const initialIndexedDB = <T = any>(database: string): ConfigStore<T> => {
     set: (key: string, value: T) => set(key, value, store),
     setMany: (values: [string, T][]) => setMany(values, store),
     keys: () => keys(store),
+    entries: () => entries(store),
     delete: (key: string) => del(key, store),
     clear: () => clear(store),
   };
@@ -27,9 +38,10 @@ const scopedIndexedDB = () => {
 
   return <T = any>(scope: string): Readonly<ConfigStore<T>> => {
     if (!storeCache.has(scope)) {
+      const prefix = `${scope}:`;
       const store = {
-        get: async (key: string) => idb.get(`${scope}:${key}`),
-        set: (key: string, value: T) => idb.set(`${scope}:${key}`, value),
+        get: async (key: string) => idb.get(prefix + key),
+        set: (key: string, value: T) => idb.set(prefix + key, value),
         setMany: (values: [string, T][]) =>
           idb.setMany(values.map(([k, v]) => [`${scope}:${k}`, v])),
         keys: () =>
@@ -37,16 +49,24 @@ const scopedIndexedDB = () => {
             .keys()
             .then(keys =>
               keys
-                .filter(k => k.startsWith(`${scope}:`))
-                .map(k => k.replace(`${scope}:`, ''))
+                .filter(k => k.startsWith(prefix))
+                .map(k => k.slice(prefix.length))
             ),
-        delete: (key: string) => idb.delete(`${scope}:${key}`),
+        entries: () =>
+          idb
+            .entries()
+            .then(entries =>
+              entries
+                .filter(([k]) => k.startsWith(prefix))
+                .map(([k, v]) => [k.slice(prefix.length), v] as [string, T])
+            ),
+        delete: (key: string) => idb.delete(prefix + key),
         clear: async () => {
           await idb
             .keys()
             .then(keys =>
               Promise.all(
-                keys.filter(k => k.startsWith(`${scope}:`)).map(k => del(k))
+                keys.filter(k => k.startsWith(prefix)).map(k => del(k))
               )
             );
         },

packages/data-center/tests/auth.spec.ts

@@ -5,9 +5,9 @@ import { getDataCenter } from './utils.js';
 import 'fake-indexeddb/auto';
 
 test.describe('Auth', () => {
-  test('signin', async () => {});
+  test('sign in', async () => {});
 
-  test('signout', async () => {});
+  test('sign out', async () => {});
 
   test('isLogin', async () => {});
 

packages/data-center/tests/cloud-sync.spec.ts

@@ -5,7 +5,7 @@ import { getDataCenter } from './utils.js';
 import 'fake-indexeddb/auto';
 
 test.describe('Cloud Sync', () => {
-  test('get cloud the sync flag of workspace  ', async () => {});
+  test('get cloud the sync flag of workspace', async () => {});
 
   test('enable [cloud sync feature]', async () => {});
 
@@ -13,7 +13,7 @@ test.describe('Cloud Sync', () => {
 
   test('editor cloud storage', async () => {});
 
-  test('cloud sync is in-progresss', async () => {});
+  test('cloud sync is in-progress', async () => {});
 
   test('cloud sync is completed', async () => {});
 

packages/data-center/tests/collaborate.spec.ts

@@ -9,7 +9,7 @@ test.describe('Collaborate', () => {
 
   test('collaborate workspace name', async () => {});
 
-  test('collaborate workspace avator', async () => {});
+  test('collaborate workspace avatar', async () => {});
 
   test('collaborate workspace list', async () => {});
 });

packages/data-center/tests/permission.spec.ts

@@ -11,7 +11,7 @@ test.describe('Permission', () => {
 
   test('make workspace private', async () => {});
 
-  test('unlogin user open the public workspace ', async () => {});
+  test('un-login user open the public workspace ', async () => {});
 
-  test('unlogin user open the private workspace ', async () => {});
+  test('un-login user open the private workspace ', async () => {});
 });

packages/data-center/tests/search.spec.ts

@@ -1,9 +1,26 @@
+import assert from 'assert';
 import { test, expect } from '@playwright/test';
 
-import { getDataCenter } from './utils.js';
+import { getDataCenter, waitOnce } from './utils.js';
 
 import 'fake-indexeddb/auto';
 
 test.describe('Search', () => {
-  test('search result', async () => {});
+  test('search result', async () => {
+    const dc = await getDataCenter();
+    const workspace = await dc.load('test');
+
+    assert(workspace);
+    workspace.createPage('test');
+    await waitOnce(workspace.signals.pageAdded);
+    const page = workspace.getPage('test');
+    assert(page);
+
+    const text = new page.Text(page, 'hello world');
+    const blockId = page.addBlock({ flavour: 'affine:paragraph', text });
+
+    expect(workspace.search('hello')).toStrictEqual(
+      new Map([[blockId, 'test']])
+    );
+  });
 });

packages/data-center/tests/utils.ts

@@ -1,5 +1,9 @@
-export const getDataCenter = () => {
-  return import('../src/index.js').then(async dataCenter =>
-    dataCenter.getDataCenter(false)
-  );
+import { Signal } from '@blocksuite/store';
+
+export const getDataCenter = async () => {
+  const dataCenter = await import('../src/index.js');
+  return await dataCenter.getDataCenter(false);
 };
+
+export const waitOnce = <T>(signal: Signal<T>) =>
+  new Promise<T>(resolve => signal.once(val => resolve(val)));

packages/data-center/tests/workspace.spec.ts

@@ -4,8 +4,18 @@ import { getDataCenter } from './utils.js';
 
 import 'fake-indexeddb/auto';
 
-test.describe('Workspace', () => {
-  test('list workspaces', async () => {
+test.describe('workspace', () => {
+  test('create', async () => {});
+
+  test('load', async () => {});
+
+  test('get workspace name', async () => {});
+  test('set workspace name', async () => {});
+
+  test('get workspace avatar', async () => {});
+  test('set workspace avatar', async () => {});
+
+  test('list', async () => {
     const dataCenter = await getDataCenter();
     await dataCenter.clear();
 
@@ -16,14 +26,23 @@ test.describe('Workspace', () => {
       dataCenter.load('test6'),
     ]);
 
-    expect(await dataCenter.list()).toStrictEqual([
-      'test3',
-      'test4',
-      'test5',
-      'test6',
-    ]);
+    expect(await dataCenter.list()).toStrictEqual({
+      test3: { local: true },
+      test4: { local: true },
+      test5: { local: true },
+      test6: { local: true },
+    });
+
+    await dataCenter.reload('test3', { providerId: 'affine' });
+    expect(await dataCenter.list()).toStrictEqual({
+      test3: { affine: true, local: true },
+      test4: { local: true },
+      test5: { local: true },
+      test6: { local: true },
+    });
   });
-  test('destroy workspaces', async () => {
+
+  test('destroy', async () => {
     const dataCenter = await getDataCenter();
     await dataCenter.clear();
 
@@ -39,23 +58,13 @@ test.describe('Workspace', () => {
     expect(ws3 !== ws4).toBeTruthy();
   });
 
-  test('remove workspaces', async () => {
+  test('remove', async () => {
     const dataCenter = await getDataCenter();
     await dataCenter.clear();
 
     // remove workspace will remove workspace data
     await Promise.all([dataCenter.load('test9'), dataCenter.load('test10')]);
     await dataCenter.delete('test9');
-    expect(await dataCenter.list()).toStrictEqual(['test10']);
+    expect(await dataCenter.list()).toStrictEqual({ test10: { local: true } });
   });
-
-  test('create workspace', async () => {});
-  test('get the workspace', async () => {});
-
-  test('get  workspace name', async () => {});
-  test('set  workspace name', async () => {});
-
-  test('get  workspace avatar', async () => {});
-
-  test('set  workspace avatar', async () => {});
 });