feat(infra): doc properties by orm (#8382)

create new orm table docConfiguration

move primary store to docConfiguration

packages/common/infra/src/modules/doc/entities/doc.ts

@@ -29,6 +29,7 @@ export class Doc extends Entity {
   public readonly record = this.scope.props.record;
 
   readonly meta$ = this.record.meta$;
+  readonly properties$ = this.record.properties$;
   readonly primaryMode$ = this.record.primaryMode$;
   readonly title$ = this.record.title$;
   readonly trash$ = this.record.trash$;

packages/common/infra/src/modules/doc/entities/property-list.ts

@@ -0,0 +1,27 @@
+import { Entity } from '../../../framework';
+import { LiveData } from '../../../livedata';
+import type { DocCustomPropertyInfo } from '../../db/schema/schema';
+import type { DocPropertiesStore } from '../stores/doc-properties';
+
+export class DocPropertyList extends Entity {
+  constructor(private readonly docPropertiesStore: DocPropertiesStore) {
+    super();
+  }
+
+  properties$ = LiveData.from(
+    this.docPropertiesStore.watchDocPropertyInfoList(),
+    []
+  );
+
+  updatePropertyInfo(id: string, properties: Partial<DocCustomPropertyInfo>) {
+    this.docPropertiesStore.updateDocPropertyInfo(id, properties);
+  }
+
+  createProperty(properties: DocCustomPropertyInfo) {
+    return this.docPropertiesStore.createDocPropertyInfo(properties);
+  }
+
+  removeProperty(id: string) {
+    this.docPropertiesStore.removeDocPropertyInfo(id);
+  }
+}

packages/common/infra/src/modules/doc/entities/record.ts

@@ -3,6 +3,8 @@ import type { DocMeta } from '@blocksuite/affine/store';
 
 import { Entity } from '../../../framework';
 import { LiveData } from '../../../livedata';
+import type { DocProperties } from '../../db';
+import type { DocPropertiesStore } from '../stores/doc-properties';
 import type { DocsStore } from '../stores/docs';
 
 /**
@@ -12,7 +14,10 @@ import type { DocsStore } from '../stores/docs';
  */
 export class DocRecord extends Entity<{ id: string }> {
   id: string = this.props.id;
-  constructor(private readonly docsStore: DocsStore) {
+  constructor(
+    private readonly docsStore: DocsStore,
+    private readonly docPropertiesStore: DocPropertiesStore
+  ) {
     super();
   }
 
@@ -21,6 +26,15 @@ export class DocRecord extends Entity<{ id: string }> {
     {}
   );
 
+  properties$ = LiveData.from<DocProperties>(
+    this.docPropertiesStore.watchDocProperties(this.id),
+    { id: this.id }
+  );
+
+  setProperties(properties: Partial<DocProperties>): void {
+    this.docPropertiesStore.updateDocProperties(this.id, properties);
+  }
+
   setMeta(meta: Partial<DocMeta>): void {
     this.docsStore.setDocMeta(this.id, meta);
   }

packages/common/infra/src/modules/doc/index.ts

@@ -6,26 +6,27 @@ export { DocService } from './services/doc';
 export { DocsService } from './services/docs';
 
 import type { Framework } from '../../framework';
-import {
-  WorkspaceLocalState,
-  WorkspaceScope,
-  WorkspaceService,
-} from '../workspace';
+import { WorkspaceDBService } from '../db';
+import { WorkspaceScope, WorkspaceService } from '../workspace';
 import { Doc } from './entities/doc';
+import { DocPropertyList } from './entities/property-list';
 import { DocRecord } from './entities/record';
 import { DocRecordList } from './entities/record-list';
 import { DocScope } from './scopes/doc';
 import { DocService } from './services/doc';
 import { DocsService } from './services/docs';
+import { DocPropertiesStore } from './stores/doc-properties';
 import { DocsStore } from './stores/docs';
 
 export function configureDocModule(framework: Framework) {
   framework
     .scope(WorkspaceScope)
     .service(DocsService, [DocsStore])
-    .store(DocsStore, [WorkspaceService, WorkspaceLocalState])
-    .entity(DocRecord, [DocsStore])
+    .store(DocPropertiesStore, [WorkspaceService, WorkspaceDBService])
+    .store(DocsStore, [WorkspaceService, DocPropertiesStore])
+    .entity(DocRecord, [DocsStore, DocPropertiesStore])
     .entity(DocRecordList, [DocsStore])
+    .entity(DocPropertyList, [DocPropertiesStore])
     .scope(DocScope)
     .entity(Doc, [DocScope, DocsStore, WorkspaceService])
     .service(DocService);

packages/common/infra/src/modules/doc/services/docs.ts

@@ -5,6 +5,7 @@ import { Service } from '../../../framework';
 import { type DocProps, initDocFromProps } from '../../../initialization';
 import { ObjectPool } from '../../../utils';
 import type { Doc } from '../entities/doc';
+import { DocPropertyList } from '../entities/property-list';
 import { DocRecordList } from '../entities/record-list';
 import { DocScope } from '../scopes/doc';
 import type { DocsStore } from '../stores/docs';
@@ -19,6 +20,8 @@ export class DocsService extends Service {
     },
   });
 
+  propertyList = this.framework.createEntity(DocPropertyList);
+
   constructor(private readonly store: DocsStore) {
     super();
   }

packages/common/infra/src/modules/doc/stores/doc-properties.ts

@@ -0,0 +1,232 @@
+import { differenceBy, isNil, omitBy } from 'lodash-es';
+import { combineLatest, map, switchMap } from 'rxjs';
+import { AbstractType as YAbstractType } from 'yjs';
+
+import { Store } from '../../../framework';
+import {
+  yjsObserveByPath,
+  yjsObserveDeep,
+} from '../../../utils/yjs-observable';
+import type { WorkspaceDBService } from '../../db';
+import type {
+  DocCustomPropertyInfo,
+  DocProperties,
+} from '../../db/schema/schema';
+import type { WorkspaceService } from '../../workspace';
+
+interface LegacyDocProperties {
+  custom?: Record<string, { value: unknown } | undefined>;
+  system?: Record<string, { value: unknown } | undefined>;
+}
+
+type LegacyDocPropertyInfo = {
+  id?: string;
+  name?: string;
+  type?: string;
+};
+
+type LegacyDocPropertyInfoList = Record<
+  string,
+  LegacyDocPropertyInfo | undefined
+>;
+
+export class DocPropertiesStore extends Store {
+  constructor(
+    private readonly workspaceService: WorkspaceService,
+    private readonly dbService: WorkspaceDBService
+  ) {
+    super();
+  }
+
+  updateDocProperties(id: string, config: Partial<DocProperties>) {
+    return this.dbService.db.docProperties.create({
+      id,
+      ...config,
+    });
+  }
+
+  getDocPropertyInfoList() {
+    const db = this.dbService.db.docCustomPropertyInfo.find();
+    const legacy = this.upgradeLegacyDocPropertyInfoList(
+      this.getLegacyDocPropertyInfoList()
+    );
+    const notOverridden = differenceBy(legacy, db, i => i.id);
+    return [...db, ...notOverridden].filter(i => !i.isDeleted);
+  }
+
+  createDocPropertyInfo(config: DocCustomPropertyInfo) {
+    return this.dbService.db.docCustomPropertyInfo.create(config).id;
+  }
+
+  removeDocPropertyInfo(id: string) {
+    this.updateDocPropertyInfo(id, {
+      additionalData: {}, // also remove additional data to reduce size
+      isDeleted: true,
+    });
+  }
+
+  updateDocPropertyInfo(id: string, config: Partial<DocCustomPropertyInfo>) {
+    const needMigration = !this.dbService.db.docCustomPropertyInfo.get(id);
+    if (needMigration) {
+      // if this property is not in db, we need to migration it from legacy to db, only type and name is needed
+      this.migrateLegacyDocPropertyInfo(id, config);
+    } else {
+      this.dbService.db.docCustomPropertyInfo.update(id, config);
+    }
+  }
+
+  migrateLegacyDocPropertyInfo(
+    id: string,
+    override: Partial<DocCustomPropertyInfo>
+  ) {
+    const legacy = this.getLegacyDocPropertyInfo(id);
+    this.dbService.db.docCustomPropertyInfo.create({
+      id,
+      type:
+        legacy?.type ??
+        'unknown' /* should never reach here, just for safety, we need handle unknown property type */,
+      name: legacy?.name,
+      ...override,
+    });
+  }
+
+  watchDocPropertyInfoList() {
+    return combineLatest([
+      this.watchLegacyDocPropertyInfoList().pipe(
+        map(this.upgradeLegacyDocPropertyInfoList)
+      ),
+      this.dbService.db.docCustomPropertyInfo.find$({}),
+    ]).pipe(
+      map(([legacy, db]) => {
+        const notOverridden = differenceBy(legacy, db, i => i.id);
+        return [...db, ...notOverridden].filter(i => !i.isDeleted);
+      })
+    );
+  }
+
+  getDocProperties(id: string) {
+    return {
+      ...this.upgradeLegacyDocProperties(this.getLegacyDocProperties(id)),
+      ...omitBy(this.dbService.db.docProperties.get(id), isNil),
+      // db always override legacy, but nil value should not override
+    };
+  }
+
+  watchDocProperties(id: string) {
+    return combineLatest([
+      this.watchLegacyDocProperties(id).pipe(
+        map(this.upgradeLegacyDocProperties)
+      ),
+      this.dbService.db.docProperties.get$(id),
+    ]).pipe(
+      map(
+        ([legacy, db]) =>
+          ({
+            ...legacy,
+            ...omitBy(db, isNil), // db always override legacy, but nil value should not override
+          }) as DocProperties
+      )
+    );
+  }
+
+  private upgradeLegacyDocProperties(properties?: LegacyDocProperties) {
+    if (!properties) {
+      return {};
+    }
+    const newProperties: Record<string, unknown> = {};
+    for (const [key, info] of Object.entries(properties.system ?? {})) {
+      if (info?.value !== undefined) {
+        newProperties[key] = info.value;
+      }
+    }
+    for (const [key, info] of Object.entries(properties.custom ?? {})) {
+      if (info?.value !== undefined) {
+        newProperties['custom:' + key] = info.value;
+      }
+    }
+    return newProperties;
+  }
+
+  private upgradeLegacyDocPropertyInfoList(
+    infoList?: LegacyDocPropertyInfoList
+  ) {
+    if (!infoList) {
+      return [];
+    }
+
+    const newInfoList: DocCustomPropertyInfo[] = [];
+
+    for (const [id, info] of Object.entries(infoList ?? {})) {
+      if (info?.type) {
+        newInfoList.push({
+          id,
+          name: info.name,
+          type: info.type,
+        });
+      }
+    }
+
+    return newInfoList;
+  }
+
+  private getLegacyDocProperties(id: string) {
+    return this.workspaceService.workspace.rootYDoc
+      .getMap<any>('affine:workspace-properties')
+      .get('pageProperties')
+      ?.get(id)
+      ?.toJSON() as LegacyDocProperties | undefined;
+  }
+
+  private watchLegacyDocProperties(id: string) {
+    return yjsObserveByPath(
+      this.workspaceService.workspace.rootYDoc.getMap<any>(
+        'affine:workspace-properties'
+      ),
+      `pageProperties.${id}`
+    ).pipe(
+      switchMap(yjsObserveDeep),
+      map(
+        p =>
+          (p instanceof YAbstractType ? p.toJSON() : p) as
+            | LegacyDocProperties
+            | undefined
+      )
+    );
+  }
+
+  private getLegacyDocPropertyInfoList() {
+    return this.workspaceService.workspace.rootYDoc
+      .getMap<any>('affine:workspace-properties')
+      .get('schema')
+      ?.get('pageProperties')
+      ?.get('custom')
+      ?.toJSON() as LegacyDocPropertyInfoList | undefined;
+  }
+
+  private watchLegacyDocPropertyInfoList() {
+    return yjsObserveByPath(
+      this.workspaceService.workspace.rootYDoc.getMap<any>(
+        'affine:workspace-properties'
+      ),
+      'schema.pageProperties.custom'
+    ).pipe(
+      switchMap(yjsObserveDeep),
+      map(
+        p =>
+          (p instanceof YAbstractType ? p.toJSON() : p) as
+            | LegacyDocPropertyInfoList
+            | undefined
+      )
+    );
+  }
+
+  private getLegacyDocPropertyInfo(id: string) {
+    return this.workspaceService.workspace.rootYDoc
+      .getMap<any>('affine:workspace-properties')
+      .get('schema')
+      ?.get('pageProperties')
+      ?.get('custom')
+      ?.get(id)
+      ?.toJSON() as LegacyDocPropertyInfo | undefined;
+  }
+}

packages/common/infra/src/modules/doc/stores/docs.ts

@@ -1,15 +1,17 @@
 import type { DocMode } from '@blocksuite/affine/blocks';
 import type { DocMeta } from '@blocksuite/affine/store';
-import { isEqual } from 'lodash-es';
-import { distinctUntilChanged, Observable } from 'rxjs';
+import { distinctUntilChanged, map, switchMap } from 'rxjs';
+import { Array as YArray, Map as YMap } from 'yjs';
 
 import { Store } from '../../../framework';
-import type { WorkspaceLocalState, WorkspaceService } from '../../workspace';
+import { yjsObserve, yjsObserveByPath, yjsObserveDeep } from '../../../utils';
+import type { WorkspaceService } from '../../workspace';
+import type { DocPropertiesStore } from './doc-properties';
 
 export class DocsStore extends Store {
   constructor(
     private readonly workspaceService: WorkspaceService,
-    private readonly localState: WorkspaceLocalState
+    private readonly docPropertiesStore: DocPropertiesStore
   ) {
     super();
   }
@@ -23,72 +25,67 @@ export class DocsStore extends Store {
   }
 
   watchDocIds() {
-    return new Observable<string[]>(subscriber => {
-      const emit = () => {
-        subscriber.next(
-          this.workspaceService.workspace.docCollection.meta.docMetas.map(
-            v => v.id
-          )
-        );
-      };
-
-      emit();
-
-      const dispose =
-        this.workspaceService.workspace.docCollection.meta.docMetaUpdated.on(
-          emit
-        ).dispose;
-      return () => {
-        dispose();
-      };
-    });
+    return yjsObserveByPath(
+      this.workspaceService.workspace.rootYDoc.getMap('meta'),
+      'pages'
+    ).pipe(
+      switchMap(yjsObserve),
+      map(meta => {
+        if (meta instanceof YArray) {
+          return meta.map(v => v.get('id'));
+        } else {
+          return [];
+        }
+      })
+    );
   }
 
   watchTrashDocIds() {
-    return new Observable<string[]>(subscriber => {
-      const emit = () => {
-        subscriber.next(
-          this.workspaceService.workspace.docCollection.meta.docMetas
-            .map(v => (v.trash ? v.id : null))
-            .filter(Boolean) as string[]
-        );
-      };
-
-      emit();
-
-      const dispose =
-        this.workspaceService.workspace.docCollection.meta.docMetaUpdated.on(
-          emit
-        ).dispose;
-      return () => {
-        dispose();
-      };
-    });
+    return yjsObserveByPath(
+      this.workspaceService.workspace.rootYDoc.getMap('meta'),
+      'pages'
+    ).pipe(
+      switchMap(yjsObserveDeep),
+      map(meta => {
+        if (meta instanceof YArray) {
+          return meta
+            .map(v => (v.get('trash') ? v.get('id') : null))
+            .filter(Boolean) as string[];
+        } else {
+          return [];
+        }
+      })
+    );
   }
 
   watchDocMeta(id: string) {
-    let meta: DocMeta | null = null;
-    return new Observable<Partial<DocMeta>>(subscriber => {
-      const emit = () => {
-        if (meta === null) {
-          // getDocMeta is heavy, so we cache the doc meta reference
-          meta =
-            this.workspaceService.workspace.docCollection.meta.getDocMeta(id) ||
-            null;
+    return yjsObserveByPath(
+      this.workspaceService.workspace.rootYDoc.getMap('meta'),
+      'pages'
+    ).pipe(
+      switchMap(yjsObserve),
+      map(meta => {
+        if (meta instanceof YArray) {
+          let docMetaYMap = null as YMap<any> | null;
+          meta.forEach(doc => {
+            if (doc.get('id') === id) {
+              docMetaYMap = doc;
+            }
+          });
+          return docMetaYMap;
+        } else {
+          return null;
         }
-        subscriber.next({ ...meta });
-      };
-
-      emit();
-
-      const dispose =
-        this.workspaceService.workspace.docCollection.meta.docMetaUpdated.on(
-          emit
-        ).dispose;
-      return () => {
-        dispose();
-      };
-    }).pipe(distinctUntilChanged((p, c) => isEqual(p, c)));
+      }),
+      switchMap(yjsObserveDeep),
+      map(meta => {
+        if (meta instanceof YMap) {
+          return meta.toJSON() as Partial<DocMeta>;
+        } else {
+          return {};
+        }
+      })
+    );
   }
 
   watchDocListReady() {
@@ -102,15 +99,20 @@ export class DocsStore extends Store {
   }
 
   setDocPrimaryModeSetting(id: string, mode: DocMode) {
-    return this.localState.set(`page:${id}:mode`, mode);
+    return this.docPropertiesStore.updateDocProperties(id, {
+      primaryMode: mode,
+    });
   }
 
   getDocPrimaryModeSetting(id: string) {
-    return this.localState.get<DocMode>(`page:${id}:mode`);
+    return this.docPropertiesStore.getDocProperties(id)?.primaryMode;
   }
 
   watchDocPrimaryModeSetting(id: string) {
-    return this.localState.watch<DocMode>(`page:${id}:mode`);
+    return this.docPropertiesStore.watchDocProperties(id).pipe(
+      map(config => config?.primaryMode),
+      distinctUntilChanged((p, c) => p === c)
+    );
   }
 
   waitForDocLoadReady(id: string) {