Merge branch 'master'

Conflicts:
	.vscode/settings.json
	package.json
	packages/data-center/package.json
	pnpm-lock.yaml

packages/data-center/package.json

@@ -6,7 +6,6 @@
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
   "exports": {
-    "./src/*": "./dist/src/*.js",
     ".": "./dist/src/index.js"
   },
   "scripts": {
@@ -23,17 +22,19 @@
     "@playwright/test": "^1.29.1",
     "@types/debug": "^4.1.7",
     "fake-indexeddb": "4.0.1",
-    "typescript": "^4.8.4",
-    "yjs": "^13.5.44"
+    "lit": "^2.6.1",
+    "typescript": "^4.9.5",
+    "yjs": "^13.5.45"
   },
   "dependencies": {
-    "@blocksuite/blocks": "0.4.0-20230111171650-bc63456",
-    "@blocksuite/store": "0.4.0-20230111171650-bc63456",
+    "@blocksuite/blocks": "0.4.0-alpha.2",
+    "@blocksuite/store": "0.4.0-alpha.2",
     "@tauri-apps/api": "^1.2.0",
     "debug": "^4.3.4",
     "encoding": "^0.1.13",
     "firebase": "^9.15.0",
     "idb-keyval": "^6.2.0",
+    "js-base64": "^3.7.5",
     "ky": "^0.33.0",
     "ky-universal": "^0.11.0",
     "lib0": "^0.2.58",

packages/data-center/src/datacenter.ts

@@ -1,6 +1,9 @@
 import { WorkspaceUnitCollection } from './workspace-unit-collection.js';
 import type { WorkspaceUnitCollectionChangeEvent } from './workspace-unit-collection';
-import { Workspace as BlocksuiteWorkspace } from '@blocksuite/store';
+import {
+  StoreOptions,
+  Workspace as BlocksuiteWorkspace,
+} from '@blocksuite/store';
 import type {
   BaseProvider,
   CreateWorkspaceInfoParams,
@@ -14,7 +17,6 @@ import { getLogger } from './logger';
 import { createBlocksuiteWorkspace } from './utils/index.js';
 import { MessageCenter } from './message';
 import { WorkspaceUnit } from './workspace-unit';
-
 /**
  * @class DataCenter
  * @classdesc Data center is made for managing different providers for business
@@ -125,12 +127,12 @@ export class DataCenter {
    * get a new workspace only has room id
    * @param {string} workspaceId workspace id
    */
-  private _getBlocksuiteWorkspace(workspaceId: string) {
+  private _getBlocksuiteWorkspace(workspaceId: string, params: StoreOptions) {
     // const workspaceInfo = this._workspaceUnitCollection.find(workspaceId);
     // assert(workspaceInfo, 'Workspace not found');
     return (
       // this._workspaceInstances.get(workspaceId) ||
-      createBlocksuiteWorkspace(workspaceId)
+      createBlocksuiteWorkspace(workspaceId, params)
     );
   }
 
@@ -171,7 +173,15 @@ export class DataCenter {
     const provider = this.providerMap.get(workspaceUnit.provider);
     assert(provider, `provide '${workspaceUnit.provider}' is not registered`);
     this._logger(`Loading ${workspaceUnit.provider} workspace: `, workspaceId);
-    const workspace = this._getBlocksuiteWorkspace(workspaceId);
+
+    const params: StoreOptions = {};
+    if (provider.id === 'affine') {
+      params.blobOptionsGetter = (k: string) =>
+        ({ api: '/api/workspace', token: provider.getToken() }[k]);
+    } else {
+      params.blobOptionsGetter = (k: string) => undefined;
+    }
+    const workspace = this._getBlocksuiteWorkspace(workspaceId, params);
     this._workspaceInstances.set(workspaceId, workspace);
     await provider.warpWorkspace(workspace);
     this._workspaceUnitCollection.workspaces.forEach(workspaceUnit => {
@@ -187,7 +197,7 @@ export class DataCenter {
     // FIXME: hard code for public workspace
     const provider = this.providerMap.get('affine');
     assert(provider);
-    const blocksuiteWorkspace = this._getBlocksuiteWorkspace(workspaceId);
+    const blocksuiteWorkspace = this._getBlocksuiteWorkspace(workspaceId, {});
     await provider.loadPublicWorkspace(blocksuiteWorkspace);
 
     const workspaceUnitForPublic = new WorkspaceUnit({

packages/data-center/src/provider/affine/affine.ts

@@ -13,7 +13,11 @@ import { getApis, Workspace } from './apis/index.js';
 import type { Apis, WorkspaceDetail, Callback } from './apis';
 import { token } from './apis/token.js';
 import { WebsocketClient } from './channel';
-import { loadWorkspaceUnit, createWorkspaceUnit } from './utils.js';
+import {
+  loadWorkspaceUnit,
+  createWorkspaceUnit,
+  migrateBlobDB,
+} from './utils.js';
 import { WorkspaceUnit } from '../../workspace-unit.js';
 import { createBlocksuiteWorkspace, applyUpdate } from '../../utils/index.js';
 import type { SyncMode } from '../../workspace-unit';
@@ -103,47 +107,59 @@ export class AffineProvider extends BaseProvider {
     metadata,
   }: ChannelMessage) {
     this._logger('receive server message');
-    const addedWorkspaces: WorkspaceUnit[] = [];
-    const removeWorkspaceList = this._workspaces.list().map(w => w.id);
+    const newlyCreatedWorkspaces: WorkspaceUnit[] = [];
+    const currentWorkspaceIds = this._workspaces.list().map(w => w.id);
+    const newlyRemovedWorkspacecIds = currentWorkspaceIds;
     for (const [id, detail] of Object.entries(ws_details)) {
       const { name, avatar } = metadata[id];
-      const index = removeWorkspaceList.indexOf(id);
-      if (index !== -1) {
-        removeWorkspaceList.splice(index, 1);
+
+      /**
+       * collect the workspaces that need to be removed in the context
+       */
+      const workspaceIndex = currentWorkspaceIds.indexOf(id);
+      const ifWorkspaceExist = workspaceIndex !== -1;
+      if (ifWorkspaceExist) {
+        newlyRemovedWorkspacecIds.splice(workspaceIndex, 1);
       }
-      assert(
-        name,
-        'workspace name not found by id when receive server message'
-      );
-      const workspace = {
-        name: name,
-        avatar,
-        owner: {
-          name: detail.owner.name,
-          id: detail.owner.id,
-          email: detail.owner.email,
-          avatar: detail.owner.avatar_url,
-        },
-        published: detail.public,
-        memberCount: detail.member_count,
-        provider: this.id,
-        syncMode: 'core' as SyncMode,
-      };
-      if (this._workspaces.get(id)) {
-        // update workspaces
-        this._workspaces.update(id, workspace);
+
+      /**
+       * if workspace name is  not empty, it is  a valid workspace, so sync its state
+       */
+      if (name) {
+        const workspace = {
+          name: name,
+          avatar,
+          owner: {
+            name: detail.owner.name,
+            id: detail.owner.id,
+            email: detail.owner.email,
+            avatar: detail.owner.avatar_url,
+          },
+          published: detail.public,
+          memberCount: detail.member_count,
+          provider: this.id,
+          syncMode: 'core' as SyncMode,
+        };
+        if (this._workspaces.get(id)) {
+          // update workspaces
+          this._workspaces.update(id, workspace);
+        } else {
+          const workspaceUnit = await loadWorkspaceUnit(
+            { id, ...workspace },
+            this._apis
+          );
+          newlyCreatedWorkspaces.push(workspaceUnit);
+        }
       } else {
-        const workspaceUnit = await loadWorkspaceUnit(
-          { id, ...workspace },
-          this._apis
-        );
-        addedWorkspaces.push(workspaceUnit);
+        console.log(`[log warn]  ${id} name is empty`);
       }
     }
-    // add workspaces
-    this._workspaces.add(addedWorkspaces);
-    // remove workspaces
-    this._workspaces.remove(removeWorkspaceList);
+
+    // sync newlyCreatedWorkspaces to context
+    this._workspaces.add(newlyCreatedWorkspaces);
+
+    // sync newlyRemoveWorkspaces to context
+    this._workspaces.remove(newlyRemovedWorkspacecIds);
   }
 
   private _getWebsocketProvider(workspace: BlocksuiteWorkspace) {
@@ -157,6 +173,8 @@ export class AffineProvider extends BaseProvider {
       }://${window.location.host}/api/sync/`;
       ws = new WebsocketProvider(wsUrl, room, doc, {
         params: { token: this._apis.token.refresh },
+        // @ts-expect-error ignore the type
+        awareness: workspace.awarenessStore.awareness,
       });
       this._wsMap.set(workspace, ws);
     }
@@ -356,6 +374,10 @@ export class AffineProvider extends BaseProvider {
     await this._apis.updateWorkspace({ id, public: isPublish });
   }
 
+  public override getToken(): string {
+    return this._apis.token.token;
+  }
+
   public override async getUserByEmail(
     workspace_id: string,
     email: string
@@ -389,6 +411,7 @@ export class AffineProvider extends BaseProvider {
       provider: this.id,
       syncMode: 'core',
     });
+    await migrateBlobDB(workspaceUnit.id, id);
 
     const blocksuiteWorkspace = createBlocksuiteWorkspace(id);
     assert(workspaceUnit.blocksuiteWorkspace);
@@ -407,7 +430,7 @@ export class AffineProvider extends BaseProvider {
     token.clear();
     this._channel?.disconnect();
     this._wsMap.forEach(ws => ws.disconnect());
-    this._workspaces.clear();
+    this._workspaces.clear(false);
     storage.removeItem('token');
   }
 

packages/data-center/src/provider/affine/apis/__tests__/token.spec.ts

@@ -0,0 +1,21 @@
+import { test, expect } from '@playwright/test';
+import { Token } from '../token.js';
+
+test.describe('class Token', () => {
+  test('parse tokens', () => {
+    const tokenString = `eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2NzU2Nzk1MjAsImlkIjo2LCJuYW1lIjoidGVzdCIsImVtYWlsIjoidGVzdEBnbWFpbC5jb20iLCJhdmF0YXJfdXJsIjoiaHR0cHM6Ly90ZXN0LmNvbS9hdmF0YXIiLCJjcmVhdGVkX2F0IjoxNjc1Njc4OTIwMzU4fQ.R8GxrNhn3gNumtapthrP6_J5eQjXLV7i-LanSPqe7hw`;
+    expect(Token.parse(tokenString)).toEqual({
+      avatar_url: 'https://test.com/avatar',
+      created_at: 1675678920358,
+      email: 'test@gmail.com',
+      exp: 1675679520,
+      id: 6,
+      name: 'test',
+    });
+  });
+
+  test('parse invalid tokens', () => {
+    const tokenString = `eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.aaa.R8GxrNhn3gNumtapthrP6_J5eQjXLV7i-LanSPqe7hw`;
+    expect(Token.parse(tokenString)).toEqual(null);
+  });
+});

packages/data-center/src/provider/affine/apis/token.ts

@@ -1,6 +1,7 @@
 import { initializeApp } from 'firebase/app';
 import { getAuth, GoogleAuthProvider, signInWithPopup } from 'firebase/auth';
 import type { User } from 'firebase/auth';
+import { decode } from 'js-base64';
 
 import { getLogger } from '../../../logger.js';
 import { bareClient } from './request.js';
@@ -31,7 +32,7 @@ type LoginResponse = {
 const login = (params: LoginParams): Promise<LoginResponse> =>
   bareClient.post('api/user/token', { json: params }).json();
 
-class Token {
+export class Token {
   private readonly _logger;
   private _accessToken!: string;
   private _refreshToken!: string;
@@ -99,21 +100,9 @@ class Token {
 
   static parse(token: string): AccessTokenMessage | null {
     try {
-      return JSON.parse(
-        String.fromCharCode.apply(
-          null,
-          Array.from(
-            Uint8Array.from(
-              window.atob(
-                // split jwt
-                token.split('.')[1]
-              ),
-              c => c.charCodeAt(0)
-            )
-          )
-        )
-      );
+      return JSON.parse(decode(token.split('.')[1]));
     } catch (error) {
+      // todo: log errors?
       return null;
     }
   }

packages/data-center/src/provider/affine/idb-kv.ts

@@ -0,0 +1,25 @@
+import { createStore, keys, setMany, getMany, clear } from 'idb-keyval';
+import * as idb from 'lib0/indexeddb.js';
+
+type IDBInstance<T = ArrayBufferLike> = {
+  keys: () => Promise<string[]>;
+  clear: () => Promise<void>;
+  deleteDB: () => Promise<void>;
+  setMany: (entries: [string, T][]) => Promise<void>;
+  getMany: (keys: string[]) => Promise<T[]>;
+};
+
+export function getDatabase<T = ArrayBufferLike>(
+  type: string,
+  database: string
+): IDBInstance<T> {
+  const name = `${database}_${type}`;
+  const db = createStore(name, type);
+  return {
+    keys: () => keys(db),
+    clear: () => clear(db),
+    deleteDB: () => idb.deleteDB(name),
+    setMany: entries => setMany(entries, db),
+    getMany: keys => getMany(keys, db),
+  };
+}

packages/data-center/src/provider/affine/utils.ts

@@ -4,6 +4,7 @@ import { createBlocksuiteWorkspace } from '../../utils/index.js';
 import type { Apis } from './apis';
 import { setDefaultAvatar } from '../utils.js';
 import { applyUpdate } from '../../utils/index.js';
+import { getDatabase } from './idb-kv.js';
 
 export const loadWorkspaceUnit = async (
   params: WorkspaceUnitCtorParams,
@@ -54,3 +55,34 @@ export const createWorkspaceUnit = async (params: WorkspaceUnitCtorParams) => {
 
   return workspaceUnit;
 };
+
+interface PendingTask {
+  id: string;
+  blob: ArrayBufferLike;
+}
+
+export const migrateBlobDB = async (
+  oldWorkspaceId: string,
+  newWorkspaceId: string
+) => {
+  const oldDB = getDatabase('blob', oldWorkspaceId);
+  const oldPendingDB = getDatabase<PendingTask>('pending', oldWorkspaceId);
+
+  const newDB = getDatabase('blob', newWorkspaceId);
+  const newPendingDB = getDatabase<PendingTask>('pending', newWorkspaceId);
+
+  const keys = await oldDB.keys();
+  const values = await oldDB.getMany(keys);
+  const entries = keys.map((key, index) => {
+    return [key, values[index]] as [string, ArrayBufferLike];
+  });
+  await newDB.setMany(entries);
+
+  const pendingEntries = entries.map(([id, blob]) => {
+    return [id, { id, blob }] as [string, PendingTask];
+  });
+  await newPendingDB.setMany(pendingEntries);
+
+  await oldDB.clear();
+  await oldPendingDB.clear();
+};

packages/data-center/src/provider/base.ts

@@ -64,6 +64,10 @@ export class BaseProvider {
     return;
   }
 
+  public getToken(): string {
+    return '';
+  }
+
   /**
    * warp workspace with provider functions
    * @param workspace

packages/data-center/src/provider/local/local.ts

@@ -17,7 +17,7 @@ const WORKSPACE_KEY = 'workspaces';
 
 export class LocalProvider extends BaseProvider {
   public id = 'local';
-  private _idbMap: Map<string, IndexedDBProvider> = new Map();
+  private _idbMap: Map<BlocksuiteWorkspace, IndexedDBProvider> = new Map();
 
   constructor(params: ProviderConstructorParams) {
     super(params);
@@ -36,11 +36,11 @@ export class LocalProvider extends BaseProvider {
 
   public override async linkLocal(workspace: BlocksuiteWorkspace) {
     assert(workspace.room);
-    let idb = this._idbMap.get(workspace.room);
+    let idb = this._idbMap.get(workspace);
     if (!idb) {
       idb = new IndexedDBProvider(workspace.room, workspace.doc);
     }
-    this._idbMap.set(workspace.room, idb);
+    this._idbMap.set(workspace, idb);
     this._logger('Local data loaded');
     return workspace;
   }
@@ -79,6 +79,9 @@ export class LocalProvider extends BaseProvider {
       IndexedDBProvider.delete(id);
       this._workspaces.remove(id);
       this._storeWorkspaces(this._workspaces.list());
+      if (workspace.blocksuiteWorkspace) {
+        this._idbMap.delete(workspace.blocksuiteWorkspace);
+      }
     } else {
       this._logger(`Failed to delete workspace ${id}`);
     }
@@ -115,5 +118,6 @@ export class LocalProvider extends BaseProvider {
     workspaces.forEach(ws => IndexedDBProvider.delete(ws.id));
     this._storeWorkspaces([]);
     this._workspaces.clear();
+    this._idbMap.clear();
   }
 }

packages/data-center/src/utils/index.ts

@@ -1,10 +1,19 @@
-import { Workspace as BlocksuiteWorkspace } from '@blocksuite/store';
-import { BlockSchema } from '@blocksuite/blocks/models';
+import {
+  StoreOptions,
+  Workspace as BlocksuiteWorkspace,
+} from '@blocksuite/store';
+import { builtInSchemas, __unstableSchemas } from '@blocksuite/blocks/models';
 
-export const createBlocksuiteWorkspace = (workspaceId: string) => {
+export const createBlocksuiteWorkspace = (
+  workspaceId: string,
+  workspaceOption?: StoreOptions
+) => {
   return new BlocksuiteWorkspace({
     room: workspaceId,
-  }).register(BlockSchema);
+    ...workspaceOption,
+  })
+    .register(builtInSchemas)
+    .register(__unstableSchemas);
 };
 
 const DefaultHeadImgColors = [

packages/data-center/src/workspace-unit-collection.spec.ts

@@ -55,7 +55,7 @@ test.describe.serial('workspace meta collection observable', () => {
     workspaceUnitCollection.once(
       'change',
       (event: WorkspaceUnitCollectionChangeEvent) => {
-        expect(event.deleted?.id).toEqual('123');
+        expect(event.deleted?.[0]?.id).toEqual('123');
       }
     );
     scope.remove('123');

packages/data-center/src/workspace-unit-collection.ts

@@ -8,8 +8,8 @@ export interface WorkspaceUnitCollectionScope {
   get: (workspaceId: string) => WorkspaceUnit | undefined;
   list: () => WorkspaceUnit[];
   add: (workspace: WorkspaceUnit | WorkspaceUnit[]) => void;
-  remove: (workspaceId: string | string[]) => boolean;
-  clear: () => void;
+  remove: (workspaceId: string | string[], isUpdate?: boolean) => boolean;
+  clear: (isUpdate?: boolean) => void;
   update: (
     workspaceId: string,
     workspaceUnit: UpdateWorkspaceUnitParams
@@ -85,7 +85,7 @@ export class WorkspaceUnitCollection {
       ]);
     };
 
-    const remove = (workspaceId: string | string[]) => {
+    const remove = (workspaceId: string | string[], isUpdate = true) => {
       const workspaceIds = Array.isArray(workspaceId)
         ? workspaceId
         : [workspaceId];
@@ -111,18 +111,19 @@ export class WorkspaceUnitCollection {
       if (!workspaceUnits.length) {
         return false;
       }
-
-      this._events.emit('change', [
-        {
-          deleted: workspaceUnits,
-        } as WorkspaceUnitCollectionChangeEvent,
-      ]);
+      if (isUpdate) {
+        this._events.emit('change', [
+          {
+            deleted: workspaceUnits,
+          } as WorkspaceUnitCollectionChangeEvent,
+        ]);
+      }
 
       return true;
     };
 
-    const clear = () => {
-      remove(Array.from(scopedWorkspaceIds));
+    const clear = (isUpdate = true) => {
+      remove(Array.from(scopedWorkspaceIds), isUpdate);
     };
 
     const update = (workspaceId: string, meta: UpdateWorkspaceUnitParams) => {