refactor(core): doc property (#8465)

doc property upgraded to use orm.

The visibility of the property are simplified to three types: `always show`, `always hide`, `hide when empty`, and the default is `always show`.

![CleanShot 2024-10-14 at 15 34 52](https://github.com/user-attachments/assets/748b8b80-061f-4d6a-8579-52e59df717c2)

Added a sidebar view to manage properties
![CleanShot 2024-10-14 at 15 35 58](https://github.com/user-attachments/assets/bffa9b1a-a1a5-4708-b2e8-4963120f3af9)

new property ui in workspace settings
![CleanShot 2024-10-14 at 15 36 44](https://github.com/user-attachments/assets/572d8dcc-9b3d-462a-9bcc-5f5fa8e622da)

Property lists can be collapsed
![CleanShot 2024-10-14 at 15 37 59](https://github.com/user-attachments/assets/2b20be1a-8141-478a-8fe7-405aff6d04fd)

packages/common/infra/package.json

@@ -17,6 +17,7 @@
     "@blocksuite/affine": "0.17.18",
     "@datastructures-js/binary-search-tree": "^5.3.2",
     "foxact": "^0.2.33",
+    "fractional-indexing": "^3.2.0",
     "fuse.js": "^7.0.0",
     "graphemer": "^1.4.0",
     "idb": "^8.0.0",

packages/common/infra/src/modules/db/index.ts

@@ -4,7 +4,7 @@ import { WorkspaceDB } from './entities/db';
 import { WorkspaceDBTable } from './entities/table';
 import { WorkspaceDBService } from './services/db';
 
-export type { DocProperties } from './schema';
+export type { DocCustomPropertyInfo, DocProperties } from './schema';
 export { WorkspaceDBService } from './services/db';
 export { transformWorkspaceDBLocalToCloud } from './services/db';
 

packages/common/infra/src/modules/db/schema/index.ts

@@ -1,4 +1,4 @@
-export type { DocProperties } from './schema';
+export type { DocCustomPropertyInfo, DocProperties } from './schema';
 export {
   AFFiNE_WORKSPACE_DB_SCHEMA,
   AFFiNE_WORKSPACE_USERDATA_DB_SCHEMA,

packages/common/infra/src/modules/db/schema/schema.ts

@@ -23,6 +23,7 @@ export const AFFiNE_WORKSPACE_DB_SCHEMA = {
     type: f.string(),
     show: f.string().optional(),
     index: f.string().optional(),
+    icon: f.string().optional(),
     additionalData: f.json().optional(),
     isDeleted: f.boolean().optional(),
     // we will keep deleted properties in the database, for override legacy data

packages/common/infra/src/modules/doc/constants.ts

@@ -0,0 +1,11 @@
+import type { DocCustomPropertyInfo } from '../db';
+
+/**
+ * default built-in custom property, user can update and delete them
+ */
+export const BUILT_IN_CUSTOM_PROPERTY_TYPE = [
+  {
+    id: 'tags',
+    type: 'tags',
+  },
+] as DocCustomPropertyInfo[];

packages/common/infra/src/modules/doc/entities/doc.ts

@@ -34,6 +34,14 @@ export class Doc extends Entity {
   readonly title$ = this.record.title$;
   readonly trash$ = this.record.trash$;
 
+  customProperty$(propertyId: string) {
+    return this.record.customProperty$(propertyId);
+  }
+
+  setCustomProperty(propertyId: string, value: string) {
+    return this.record.setCustomProperty(propertyId, value);
+  }
+
   setPrimaryMode(mode: DocMode) {
     return this.record.setPrimaryMode(mode);
   }

packages/common/infra/src/modules/doc/entities/property-list.ts

@@ -1,5 +1,6 @@
 import { Entity } from '../../../framework';
 import { LiveData } from '../../../livedata';
+import { generateFractionalIndexingKeyBetween } from '../../../utils';
 import type { DocCustomPropertyInfo } from '../../db/schema/schema';
 import type { DocPropertiesStore } from '../stores/doc-properties';
 
@@ -13,15 +14,61 @@ export class DocPropertyList extends Entity {
     []
   );
 
+  sortedProperties$ = this.properties$.map(list =>
+    // default index key is '', so always before any others
+    list.toSorted((a, b) => ((a.index ?? '') > (b.index ?? '') ? 1 : -1))
+  );
+
+  propertyInfo$(id: string) {
+    return this.properties$.map(list => list.find(info => info.id === id));
+  }
+
   updatePropertyInfo(id: string, properties: Partial<DocCustomPropertyInfo>) {
     this.docPropertiesStore.updateDocPropertyInfo(id, properties);
   }
 
-  createProperty(properties: DocCustomPropertyInfo) {
+  createProperty(
+    properties: Omit<DocCustomPropertyInfo, 'id'> & { id?: string }
+  ) {
     return this.docPropertiesStore.createDocPropertyInfo(properties);
   }
 
   removeProperty(id: string) {
     this.docPropertiesStore.removeDocPropertyInfo(id);
   }
+
+  indexAt(at: 'before' | 'after', targetId?: string) {
+    const sortedChildren = this.sortedProperties$.value.filter(
+      node => node.index
+    ) as (DocCustomPropertyInfo & { index: string })[];
+    const targetIndex = targetId
+      ? sortedChildren.findIndex(node => node.id === targetId)
+      : -1;
+    if (targetIndex === -1) {
+      if (at === 'before') {
+        const first = sortedChildren.at(0);
+        return generateFractionalIndexingKeyBetween(null, first?.index ?? null);
+      } else {
+        const last = sortedChildren.at(-1);
+        return generateFractionalIndexingKeyBetween(last?.index ?? null, null);
+      }
+    } else {
+      const target = sortedChildren[targetIndex];
+      const before: DocCustomPropertyInfo | null =
+        sortedChildren[targetIndex - 1] || null;
+      const after: DocCustomPropertyInfo | null =
+        sortedChildren[targetIndex + 1] || null;
+      if (at === 'before') {
+        return generateFractionalIndexingKeyBetween(
+          before?.index ?? null,
+          target.index
+        );
+      } else {
+        return generateFractionalIndexingKeyBetween(
+          target.index,
+          after?.index ?? null
+        );
+      }
+    }
+  }
 }

packages/common/infra/src/modules/doc/entities/record.ts

@@ -31,8 +31,16 @@ export class DocRecord extends Entity<{ id: string }> {
     { id: this.id }
   );
 
-  setProperties(properties: Partial<DocProperties>): void {
-    this.docPropertiesStore.updateDocProperties(this.id, properties);
+  customProperty$(propertyId: string) {
+    return this.properties$.selector(
+      p => p['custom:' + propertyId]
+    ) as LiveData<string | undefined | null>;
+  }
+
+  setCustomProperty(propertyId: string, value: string) {
+    this.docPropertiesStore.updateDocProperties(this.id, {
+      ['custom:' + propertyId]: value,
+    });
   }
 
   setMeta(meta: Partial<DocMeta>): void {

packages/common/infra/src/modules/doc/stores/doc-properties.ts

@@ -13,6 +13,7 @@ import type {
   DocProperties,
 } from '../../db/schema/schema';
 import type { WorkspaceService } from '../../workspace';
+import { BUILT_IN_CUSTOM_PROPERTY_TYPE } from '../constants';
 
 interface LegacyDocProperties {
   custom?: Record<string, { value: unknown } | undefined>;
@@ -23,6 +24,7 @@ type LegacyDocPropertyInfo = {
   id?: string;
   name?: string;
   type?: string;
+  icon?: string;
 };
 
 type LegacyDocPropertyInfoList = Record<
@@ -50,11 +52,18 @@ export class DocPropertiesStore extends Store {
     const legacy = this.upgradeLegacyDocPropertyInfoList(
       this.getLegacyDocPropertyInfoList()
     );
-    const notOverridden = differenceBy(legacy, db, i => i.id);
-    return [...db, ...notOverridden].filter(i => !i.isDeleted);
+    const builtIn = BUILT_IN_CUSTOM_PROPERTY_TYPE;
+    const withLegacy = [...db, ...differenceBy(legacy, db, i => i.id)];
+    const all = [
+      ...withLegacy,
+      ...differenceBy(builtIn, withLegacy, i => i.id),
+    ];
+    return all.filter(i => !i.isDeleted);
   }
 
-  createDocPropertyInfo(config: DocCustomPropertyInfo) {
+  createDocPropertyInfo(
+    config: Omit<DocCustomPropertyInfo, 'id'> & { id?: string }
+  ) {
     return this.dbService.db.docCustomPropertyInfo.create(config).id;
   }
 
@@ -67,7 +76,11 @@ export class DocPropertiesStore extends Store {
 
   updateDocPropertyInfo(id: string, config: Partial<DocCustomPropertyInfo>) {
     const needMigration = !this.dbService.db.docCustomPropertyInfo.get(id);
-    if (needMigration) {
+    const isBuiltIn =
+      needMigration && BUILT_IN_CUSTOM_PROPERTY_TYPE.some(i => i.id === id);
+    if (isBuiltIn) {
+      this.createPropertyFromBuiltIn(id, config);
+    } else if (needMigration) {
       // if this property is not in db, we need to migration it from legacy to db, only type and name is needed
       this.migrateLegacyDocPropertyInfo(id, config);
     } else {
@@ -90,16 +103,32 @@ export class DocPropertiesStore extends Store {
     });
   }
 
+  createPropertyFromBuiltIn(
+    id: string,
+    override: Partial<DocCustomPropertyInfo>
+  ) {
+    const builtIn = BUILT_IN_CUSTOM_PROPERTY_TYPE.find(i => i.id === id);
+    if (!builtIn) {
+      return;
+    }
+    this.createDocPropertyInfo({ ...builtIn, ...override });
+  }
+
   watchDocPropertyInfoList() {
     return combineLatest([
       this.watchLegacyDocPropertyInfoList().pipe(
         map(this.upgradeLegacyDocPropertyInfoList)
       ),
-      this.dbService.db.docCustomPropertyInfo.find$({}),
+      this.dbService.db.docCustomPropertyInfo.find$(),
     ]).pipe(
       map(([legacy, db]) => {
-        const notOverridden = differenceBy(legacy, db, i => i.id);
-        return [...db, ...notOverridden].filter(i => !i.isDeleted);
+        const builtIn = BUILT_IN_CUSTOM_PROPERTY_TYPE;
+        const withLegacy = [...db, ...differenceBy(legacy, db, i => i.id)];
+        const all = [
+          ...withLegacy,
+          ...differenceBy(builtIn, withLegacy, i => i.id),
+        ];
+        return all.filter(i => !i.isDeleted);
       })
     );
   }
@@ -133,15 +162,15 @@ export class DocPropertiesStore extends Store {
     if (!properties) {
       return {};
     }
-    const newProperties: Record<string, unknown> = {};
+    const newProperties: Record<string, string> = {};
     for (const [key, info] of Object.entries(properties.system ?? {})) {
-      if (info?.value !== undefined) {
-        newProperties[key] = info.value;
+      if (info?.value !== undefined && info.value !== null) {
+        newProperties[key] = info.value.toString();
       }
     }
     for (const [key, info] of Object.entries(properties.custom ?? {})) {
-      if (info?.value !== undefined) {
-        newProperties['custom:' + key] = info.value;
+      if (info?.value !== undefined && info.value !== null) {
+        newProperties['custom:' + key] = info.value.toString();
       }
     }
     return newProperties;
@@ -162,6 +191,7 @@ export class DocPropertiesStore extends Store {
           id,
           name: info.name,
           type: info.type,
+          icon: info.icon,
         });
       }
     }

packages/common/infra/src/orm/core/validators/data.ts

@@ -6,7 +6,7 @@ import type { DataValidator } from './types';
 function inputType(val: any) {
   return val === null ||
     Array.isArray(val) ||
-    val.constructor === 'Object' ||
+    val.constructor === Object ||
     !val.constructor /* Object.create(null) */
     ? 'json'
     : typeof val;

packages/common/infra/src/utils/__tests__/fractional-indexing.spec.ts

@@ -0,0 +1,41 @@
+import { expect, test } from 'vitest';
+
+import { generateFractionalIndexingKeyBetween } from '../fractional-indexing';
+
+function gen(a: string | null, b: string | null) {
+  const result = generateFractionalIndexingKeyBetween(a, b);
+
+  expect(
+    a === null || b === null || (a < result && result < b),
+    `${a} ${b} ${result}`
+  ).toBe(true);
+  return result;
+}
+
+test('fractional-indexing', () => {
+  for (let i = 0; i < 100; i++) {
+    const set = new Set<string>();
+    let a = null;
+    let b = null;
+    for (let i = 0; i < 100; i++) {
+      const s1 = gen(a, b);
+      expect(a === null || b === null || (a < s1 && s1 < b)).toBe(true);
+      const s2 = gen(a, b);
+      expect(a === null || b === null || (a < s2 && s2 < b)).toBe(true);
+
+      if (set.has(s1) || set.has(s2) || s1 === s2) {
+        throw new Error('Duplicate key, ' + set.size + ', ' + s1 + ', ' + s2);
+        break;
+      }
+      set.add(s1);
+      set.add(s2);
+      if (s1 < s2) {
+        a = s1;
+        b = s2;
+      } else {
+        a = s2;
+        b = s1;
+      }
+    }
+  }
+});

packages/common/infra/src/utils/fractional-indexing.ts

@@ -0,0 +1,213 @@
+import { generateKeyBetween } from 'fractional-indexing';
+
+export interface SortableProvider<T, K extends string | number> {
+  getItems(): T[];
+  getItemId(item: T): K;
+  getItemOrder(item: T): string;
+  setItemOrder(item: T, order: string): void;
+}
+
+// Using fractional-indexing managing orders of items in a list
+/**
+ * @deprecated
+ */
+export function createFractionalIndexingSortableHelper<
+  T,
+  K extends string | number,
+>(provider: SortableProvider<T, K>) {
+  function getOrderedItems() {
+    return provider.getItems().sort((a, b) => {
+      const oa = provider.getItemOrder(a);
+      const ob = provider.getItemOrder(b);
+      return oa > ob ? 1 : oa < ob ? -1 : 0;
+    });
+  }
+
+  function getLargestOrder() {
+    const lastItem = getOrderedItems().at(-1);
+    return lastItem ? provider.getItemOrder(lastItem) : null;
+  }
+
+  function getSmallestOrder() {
+    const firstItem = getOrderedItems().at(0);
+    return firstItem ? provider.getItemOrder(firstItem) : null;
+  }
+
+  /**
+   * Get a new order at the end of the list
+   */
+  function getNewItemOrder() {
+    return generateKeyBetween(getLargestOrder(), null);
+  }
+
+  /**
+   * Move item from one position to another
+   *
+   * in the most common sorting case, moving over will visually place the dragging item to the target position
+   * the original item in the target position will either move up or down, depending on the direction of the drag
+   *
+   * @param fromId
+   * @param toId
+   */
+  function move(fromId: K, toId: K) {
+    const items = getOrderedItems();
+    const from = items.findIndex(i => provider.getItemId(i) === fromId);
+    const to = items.findIndex(i => provider.getItemId(i) === toId);
+    const fromItem = items[from];
+    const toItem = items[to];
+    const toNextItem = items[from < to ? to + 1 : to - 1];
+    const toOrder = toItem ? provider.getItemOrder(toItem) : null;
+    const toNextOrder = toNextItem ? provider.getItemOrder(toNextItem) : null;
+    const args: [string | null, string | null] =
+      from < to ? [toOrder, toNextOrder] : [toNextOrder, toOrder];
+    provider.setItemOrder(fromItem, generateKeyBetween(...args));
+  }
+
+  function moveTo(fromId: K, toId: K, position: 'before' | 'after') {
+    const items = getOrderedItems();
+    const from = items.findIndex(i => provider.getItemId(i) === fromId);
+    const to = items.findIndex(i => provider.getItemId(i) === toId);
+    const fromItem = items[from] as T | undefined;
+    if (fromItem === undefined) return;
+    const toItem = items[to] as T | undefined;
+    const toItemPrev = items[to - 1] as T | undefined;
+    const toItemNext = items[to + 1] as T | undefined;
+    const toItemOrder = toItem ? provider.getItemOrder(toItem) : null;
+    const toItemPrevOrder = toItemPrev
+      ? provider.getItemOrder(toItemPrev)
+      : null;
+    const toItemNextOrder = toItemNext
+      ? provider.getItemOrder(toItemNext)
+      : null;
+    if (position === 'before') {
+      provider.setItemOrder(
+        fromItem,
+        generateKeyBetween(toItemPrevOrder, toItemOrder)
+      );
+    } else {
+      provider.setItemOrder(
+        fromItem,
+        generateKeyBetween(toItemOrder, toItemNextOrder)
+      );
+    }
+  }
+
+  /**
+   * Cases example:
+   * Imagine we have the following items,  | a | b | c |
+   * 1. insertBefore('b', undefined). before is not provided, which means insert b after c
+   * | a | c |
+   *         ▴
+   *         b
+   * result: | a | c | b |
+   *
+   * 2. insertBefore('b', 'a'). insert b before a
+   * | a | c |
+   * ▴
+   * b
+   *
+   * result: | b | a | c |
+   */
+  function insertBefore(
+    id: string | number,
+    beforeId: string | number | undefined
+  ) {
+    const items = getOrderedItems();
+    // assert id is in the list
+    const item = items.find(i => provider.getItemId(i) === id);
+    if (!item) return;
+
+    const beforeItemIndex = items.findIndex(
+      i => provider.getItemId(i) === beforeId
+    );
+    const beforeItem = beforeItemIndex !== -1 ? items[beforeItemIndex] : null;
+    const beforeItemPrev = beforeItem ? items[beforeItemIndex - 1] : null;
+
+    const beforeOrder = beforeItem ? provider.getItemOrder(beforeItem) : null;
+    const beforePrevOrder = beforeItemPrev
+      ? provider.getItemOrder(beforeItemPrev)
+      : null;
+
+    provider.setItemOrder(
+      item,
+      generateKeyBetween(beforePrevOrder, beforeOrder)
+    );
+  }
+
+  return {
+    getOrderedItems,
+    getLargestOrder,
+    getSmallestOrder,
+    getNewItemOrder,
+    move,
+    moveTo,
+    insertBefore,
+  };
+}
+
+/**
+ * generate a key between a and b, the result key is always satisfied with a < result < b.
+ * the key always has a random suffix, so there is no need to worry about collision.
+ *
+ * make sure a and b are generated by this function.
+ */
+export function generateFractionalIndexingKeyBetween(
+  a: string | null,
+  b: string | null
+) {
+  const randomSize = 32;
+  function randomString(length: number = randomSize) {
+    const values = new Uint8Array(length);
+    crypto.getRandomValues(values);
+    const chars =
+      '123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+    let result = '';
+    for (let i = 0; i < length; i++) {
+      result += chars.charAt(values[i] % chars.length);
+    }
+    return result;
+  }
+
+  if (a !== null && b !== null && a >= b) {
+    throw new Error('a should be smaller than b');
+  }
+
+  // get the subkey in full key
+  // e.g.
+  // a0xxxx -> a
+  // a0x0xxxx -> a0x
+  function subkey(key: string | null) {
+    if (key === null) {
+      return null;
+    }
+    if (key.length <= randomSize + 1) {
+      // no subkey
+      return key;
+    }
+    const splitAt = key.substring(0, key.length - randomSize - 1);
+    return splitAt;
+  }
+
+  const aSubkey = subkey(a);
+  const bSubkey = subkey(b);
+
+  if (aSubkey === null && bSubkey === null) {
+    // generate a new key
+    return generateKeyBetween(null, null) + '0' + randomString();
+  } else if (aSubkey === null && bSubkey !== null) {
+    // generate a key before b
+    return generateKeyBetween(null, bSubkey) + '0' + randomString();
+  } else if (bSubkey === null && aSubkey !== null) {
+    // generate a key after a
+    return generateKeyBetween(aSubkey, null) + '0' + randomString();
+  } else if (aSubkey !== null && bSubkey !== null) {
+    // generate a key between a and b
+    if (aSubkey === bSubkey && a !== null && b !== null) {
+      // conflict, if the subkeys are the same, generate a key between fullkeys
+      return generateKeyBetween(a, b) + '0' + randomString();
+    } else {
+      return generateKeyBetween(aSubkey, bSubkey) + '0' + randomString();
+    }
+  }
+  throw new Error('Never reach here');
+}

packages/common/infra/src/utils/index.ts

@@ -1,6 +1,7 @@
 export * from './async-lock';
 export * from './async-queue';
 export * from './exhaustmap-with-trailing';
+export * from './fractional-indexing';
 export * from './merge-updates';
 export * from './object-pool';
 export * from './stable-hash';