init: the first public commit for AFFiNE

libs/datasource/commands/.babelrc

@@ -0,0 +1,10 @@
+{
+    "presets": [
+        [
+            "@nrwl/web/babel",
+            {
+                "useBuiltIns": "usage"
+            }
+        ]
+    ]
+}

libs/datasource/commands/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "extends": ["plugin:@nrwl/nx/react", "../../../.eslintrc.json"],
+    "ignorePatterns": ["!**/*"],
+    "overrides": [
+        {
+            "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.ts", "*.tsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.js", "*.jsx"],
+            "rules": {}
+        }
+    ]
+}

libs/datasource/commands/README.md

@@ -0,0 +1,11 @@
+# datasource-commands
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build datasource-commands` to build the library.
+
+## Running unit tests
+
+Run `nx test datasource-commands` to execute the unit tests via [Jest](https://jestjs.io).

libs/datasource/commands/jest.config.ts

@@ -0,0 +1,15 @@
+/* eslint-disable */
+export default {
+    displayName: 'datasource-commands',
+    preset: '../../../jest.preset.js',
+    globals: {
+        'ts-jest': {
+            tsconfig: '<rootDir>/tsconfig.spec.json',
+        },
+    },
+    transform: {
+        '^.+\\.[tj]s$': 'ts-jest',
+    },
+    moduleFileExtensions: ['ts', 'js', 'html'],
+    coverageDirectory: '../../../coverage/libs/datasource/commands',
+};

libs/datasource/commands/package.json

@@ -0,0 +1,7 @@
+{
+    "name": "@toeverything/datasource/commands",
+    "version": "0.0.1",
+    "license": "MIT",
+    "type": "commonjs",
+    "dependencies": {}
+}

libs/datasource/commands/project.json

@@ -0,0 +1,33 @@
+{
+    "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "libs/datasource/commands/src",
+    "projectType": "library",
+    "tags": ["datasource:commands"],
+    "targets": {
+        "build": {
+            "executor": "@nrwl/js:tsc",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/libs/datasource/commands",
+                "main": "libs/datasource/commands/src/index.ts",
+                "tsConfig": "libs/datasource/commands/tsconfig.lib.json",
+                "assets": ["libs/datasource/commands/*.md"]
+            }
+        },
+        "lint": {
+            "executor": "@nrwl/linter:eslint",
+            "outputs": ["{options.outputFile}"],
+            "options": {
+                "lintFilePatterns": ["libs/datasource/commands/**/*.ts"]
+            }
+        },
+        "test": {
+            "executor": "@nrwl/jest:jest",
+            "outputs": ["coverage/libs/datasource/commands"],
+            "options": {
+                "jestConfig": "libs/datasource/commands/jest.config.ts",
+                "passWithNoTests": true
+            }
+        }
+    }
+}

libs/datasource/commands/src/index.ts

@@ -0,0 +1 @@
+export * from './lib/datasource-commands';

libs/datasource/commands/src/lib/commands/block-command.ts

@@ -0,0 +1,48 @@
+import { BlockFlavors } from '@toeverything/datasource/db-service';
+
+/**
+ *
+ * commands for control blocks
+ * @export
+ * @class BlockCommands
+ */
+export class BlockCommands {
+    private _workspace: string;
+    constructor(workspace: string) {
+        this._workspace = workspace;
+    }
+    /**
+     *
+     * create a block after typed id
+     * @param {keyof BlockFlavors} type
+     * @param {string} blockId
+     * @return {*}
+     * @memberof BlockCommands
+     */
+    public async createNextBlock(
+        type: keyof BlockFlavors = 'text',
+        blockId: string
+    ) {
+        // const block = await this._editor.getBlockById(blockId);
+        // if (block) {
+        //     const next_block = await this._editor.createBlock(type);
+        //     await block.after(next_block);
+        //     this._editor.selectionManager.activeNodeByNodeId(next_block.id);
+        //     return next_block;
+        // }
+        // return undefined;
+    }
+
+    /**
+     *
+     * remove block by block id
+     * @param {string} blockId
+     * @memberof BlockCommands
+     */
+    public async removeBlock(blockId: string) {
+        // const block = await this._editor.getBlockById(blockId);
+        // if (block) {
+        //     block.remove();
+        // }
+    }
+}

libs/datasource/commands/src/lib/commands/index.ts

@@ -0,0 +1,2 @@
+export * from './block-command';
+export * from './text-command';

libs/datasource/commands/src/lib/commands/text-command.ts

@@ -0,0 +1,43 @@
+/**
+ *
+ * commands for control text and text styles
+ * @export
+ * @class TextCommand
+ */
+
+export class TextCommands {
+    private _workspace: string;
+    constructor(workspace: string) {
+        this._workspace = workspace;
+    }
+
+    /**
+     *
+     * get block text by block id
+     * @param {string} blockId
+     * @return {*}
+     * @memberof TextCommand
+     */
+    public async getBlockText(blockId: string) {
+        // let string = '';
+        // const block = await this._editor.getBlockById(blockId);
+        // if (block) {
+        //     string = block.getProperty('text')?.value[0]?.text || '';
+        // }
+        // return string;
+    }
+
+    /**
+     *
+     * set block text by block id
+     * @param {string} blockId
+     * @param {string} text
+     * @memberof TextCommand
+     */
+    public async setBlockText(blockId: string, text: string) {
+        // const block = await this._editor.getBlockById(blockId);
+        // if (block) {
+        //     block.setProperty('text', { value: [{ text: text }] });
+        // }
+    }
+}

libs/datasource/commands/src/lib/datasource-commands.spec.ts

@@ -0,0 +1,7 @@
+import { EditorCommands } from './datasource-commands';
+
+describe('datasourceCommands', () => {
+    // it('should work', () => {
+    //     expect(EditorCommands()).toEqual('datasource-commands');
+    // });
+});

libs/datasource/commands/src/lib/datasource-commands.ts

@@ -0,0 +1,16 @@
+import { BlockCommands, TextCommands } from './commands';
+
+/**
+ *
+ * commands for operate servers
+ * @export
+ * @class Commands
+ */
+export class Commands {
+    public blockCommands: BlockCommands;
+    public textCommands: TextCommands;
+    constructor(workspace: string) {
+        this.blockCommands = new BlockCommands(workspace);
+        this.textCommands = new TextCommands(workspace);
+    }
+}

libs/datasource/commands/tsconfig.json

@@ -0,0 +1,23 @@
+{
+    "extends": "../../../tsconfig.base.json",
+    "compilerOptions": {
+        "module": "commonjs",
+        "forceConsistentCasingInFileNames": true,
+        "strict": true,
+        "noImplicitOverride": true,
+        "noPropertyAccessFromIndexSignature": true,
+        "noImplicitReturns": true,
+        "noFallthroughCasesInSwitch": true,
+        "strictNullChecks": true
+    },
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        },
+        {
+            "path": "./tsconfig.spec.json"
+        }
+    ]
+}

libs/datasource/commands/tsconfig.lib.json

@@ -0,0 +1,10 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "declaration": true,
+        "types": []
+    },
+    "include": ["**/*.ts"],
+    "exclude": ["jest.config.ts", "**/*.spec.ts", "**/*.test.ts"]
+}

libs/datasource/commands/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "module": "commonjs",
+        "types": ["jest", "node"]
+    },
+    "include": ["jest.config.ts", "**/*.test.ts", "**/*.spec.ts", "**/*.d.ts"]
+}

libs/datasource/db-service/.babelrc

@@ -0,0 +1,12 @@
+{
+    "presets": [
+        [
+            "@nrwl/react/babel",
+            {
+                "runtime": "automatic",
+                "useBuiltIns": "usage"
+            }
+        ]
+    ],
+    "plugins": []
+}

libs/datasource/db-service/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "extends": ["plugin:@nrwl/nx/react", "../../../.eslintrc.json"],
+    "ignorePatterns": ["!**/*"],
+    "overrides": [
+        {
+            "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.ts", "*.tsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.js", "*.jsx"],
+            "rules": {}
+        }
+    ]
+}

libs/datasource/db-service/README.md

@@ -0,0 +1,7 @@
+# datasource-db-service
+
+This library was generated with [Nx](https://nx.dev).
+
+## Running unit tests
+
+Run `nx test datasource-db-service` to execute the unit tests via [Jest](https://jestjs.io).

libs/datasource/db-service/jest.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+    displayName: 'datasource-db-service',
+    preset: '../../../jest.preset.js',
+    transform: {
+        '^.+\\.[tj]sx?$': 'babel-jest',
+    },
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+    coverageDirectory: '../../../coverage/libs/datasource/db-service',
+};

libs/datasource/db-service/package.json

@@ -0,0 +1,12 @@
+{
+    "name": "@toeverything/datasource/db-service",
+    "version": "0.0.1",
+    "license": "MIT",
+    "dependencies": {
+        "diff": "^5.1.0",
+        "nanoid": "^4.0.0"
+    },
+    "devDependencies": {
+        "@types/diff": "^5.0.2"
+    }
+}

libs/datasource/db-service/project.json

@@ -0,0 +1,44 @@
+{
+    "sourceRoot": "libs/datasource/db-service/src",
+    "projectType": "library",
+    "tags": ["datasource:db-services"],
+    "targets": {
+        "build": {
+            "executor": "@nrwl/web:rollup",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/libs/datasource/db-service",
+                "tsConfig": "libs/datasource/db-service/tsconfig.lib.json",
+                "project": "libs/datasource/db-service/package.json",
+                "entryFile": "libs/datasource/db-service/src/index.ts",
+                "external": ["react/jsx-runtime"],
+                "rollupConfig": "@nrwl/react/plugins/bundle-rollup",
+                "compiler": "babel",
+                "assets": [
+                    {
+                        "glob": "libs/datasource/db-service/README.md",
+                        "input": ".",
+                        "output": "."
+                    }
+                ]
+            }
+        },
+        "lint": {
+            "executor": "@nrwl/linter:eslint",
+            "outputs": ["{options.outputFile}"],
+            "options": {
+                "lintFilePatterns": [
+                    "libs/datasource/db-service/**/*.{ts,tsx,js,jsx}"
+                ]
+            }
+        },
+        "test": {
+            "executor": "@nrwl/jest:jest",
+            "outputs": ["coverage/libs/datasource/db-service"],
+            "options": {
+                "jestConfig": "libs/datasource/db-service/jest.config.js",
+                "passWithNoTests": true
+            }
+        }
+    }
+}

libs/datasource/db-service/src/index.ts

@@ -0,0 +1,62 @@
+import { diContainer, serviceMapByCallName } from './services';
+import type { DbServicesMap } from './services';
+export type { Template } from './services/editor-block/templates/types';
+export {
+    TemplateFactory,
+    type TemplateMeta,
+} from './services/editor-block/templates';
+
+export type { ReturnUnobserve } from './services/database';
+export type { Comment, CommentReply } from './services/comment/types';
+
+const api = new Proxy<DbServicesMap>({} as DbServicesMap, {
+    get(target, prop) {
+        const token = serviceMapByCallName[prop as string]?.token;
+        if (!token) {
+            return undefined;
+        }
+        return diContainer.getDependency(token);
+    },
+});
+
+export const services = {
+    api,
+};
+(window as any)['services'] = services;
+
+export type {
+    CreateEditorBlock,
+    ReturnEditorBlock,
+    GetEditorBlock,
+    UpdateEditorBlock,
+    DeleteEditorBlock,
+    BlockFlavors,
+    BlockFlavorKeys,
+    Column,
+    ContentColumn,
+    NumberColumn,
+    EnumColumn,
+    DateColumn,
+    BooleanColumn,
+    FileColumn,
+    DefaultColumnsValue,
+    ContentColumnValue,
+    NumberColumnValue,
+    EnumColumnValue,
+    BooleanColumnValue,
+    DateColumnValue,
+    FileColumnValue,
+    StringColumnValue,
+} from './services';
+export {
+    ColumnType,
+    isBooleanColumn,
+    isContentColumn,
+    isDateColumn,
+    isFileColumn,
+    isNumberColumn,
+    isEnumColumn,
+    isStringColumn,
+} from './services';
+export { Protocol } from './protocol';
+export { DEFAULT_COLUMN_KEYS } from './services/editor-block/utils/column/default-config';

libs/datasource/db-service/src/protocol/index.ts

@@ -0,0 +1,45 @@
+const Protocol = {
+    Block: {
+        Type: {
+            workspace: 'workspace',
+            page: 'page',
+            group: 'group',
+            title: 'title',
+            text: 'text',
+            heading1: 'heading1',
+            heading2: 'heading2',
+            heading3: 'heading3',
+            code: 'code',
+            todo: 'todo',
+            comments: 'comments',
+            tag: 'tag',
+            reference: 'reference',
+            image: 'image',
+            file: 'file',
+            audio: 'audio',
+            video: 'video',
+            shape: 'shape',
+            quote: 'quote',
+            toc: 'toc',
+            database: 'database',
+            whiteboard: 'whiteboard',
+            template: 'template',
+            discussion: 'discussion',
+            comment: 'comment',
+            activity: 'activity',
+            bullet: 'bullet',
+            numbered: 'numbered',
+            toggle: 'toggle',
+            callout: 'callout',
+            divider: 'divider',
+            groupDivider: 'groupDivider',
+            youtube: 'youtube',
+            figma: 'figma',
+            embedLink: 'embedLink',
+            grid: 'grid',
+            gridItem: 'gridItem',
+        },
+    },
+} as const;
+
+export { Protocol };

libs/datasource/db-service/src/services/base.ts

@@ -0,0 +1,168 @@
+import {
+    BlockClientInstance,
+    BlockInitOptions,
+    BlockImplInstance,
+    BlockMatcher,
+    BlockContentExporter,
+    QueryIndexMetadata,
+} from '@toeverything/datasource/jwt';
+import { DependencyCallOrConstructProps } from '@toeverything/utils';
+import { Database } from './database';
+import type { ObserveCallback, ReturnUnobserve } from './database/observer';
+
+export abstract class ServiceBaseClass {
+    protected database: Database;
+    protected get_dependency: DependencyCallOrConstructProps['getDependency'];
+    constructor(props: DependencyCallOrConstructProps) {
+        this.get_dependency = props.getDependency;
+        this.database = this.get_dependency(Database);
+    }
+
+    async getWorkspaceDbBlock(workspace: string, options?: BlockInitOptions) {
+        const db = await this.database.getDatabase(workspace, options);
+        return db.getWorkspace();
+    }
+
+    async onHistoryChange(
+        workspace: string,
+        name: string,
+        callback: (meta: Map<string, any>) => void
+    ) {
+        const db = await this.database.getDatabase(workspace);
+        db.history.onPush(name, callback);
+    }
+
+    async onHistoryRevoke(
+        workspace: string,
+        name: string,
+        callback: (meta: Map<string, any>) => void
+    ) {
+        const db = await this.database.getDatabase(workspace);
+        db.history.onPop(name, callback);
+    }
+
+    async undo(workspace: string) {
+        const db = await this.database.getDatabase(workspace);
+        return db.history.undo();
+    }
+
+    async redo(workspace: string) {
+        const db = await this.database.getDatabase(workspace);
+        return db.history.redo();
+    }
+
+    async search(
+        workspace: string,
+        query: Parameters<BlockClientInstance['searchPages']>[0]
+    ) {
+        const db = await this.database.getDatabase(workspace);
+        return db.searchPages(query);
+    }
+
+    async query(
+        workspace: string,
+        query: QueryIndexMetadata
+    ): Promise<null | any[]> {
+        const db = await this.database.getDatabase(workspace);
+        return db.query(query);
+    }
+
+    async clearUndoRedo(workspace: string) {
+        const db = await this.database.getDatabase(workspace);
+        return db.history.clear();
+    }
+
+    /**
+     * Get the block, unlike db.get, if the id does not exist, it will return undefined
+     * @param workspace
+     * @param blockId
+     * @returns
+     */
+    async getBlock(
+        workspace: string,
+        blockId: string
+    ): Promise<BlockImplInstance | undefined> {
+        const db = await this.database.getDatabase(workspace);
+        const db_block = await db.get(blockId as 'block');
+        if (db_block.id !== blockId) {
+            return undefined;
+        }
+        return db_block;
+    }
+
+    async registerContentExporter(
+        workspace: string,
+        name: string,
+        matcher: BlockMatcher,
+        exporter: BlockContentExporter
+    ) {
+        await this.database.registerContentExporter(
+            workspace,
+            name,
+            matcher,
+            exporter
+        );
+    }
+
+    async unregisterContentExporter(workspace: string, name: string) {
+        await this.database.unregisterContentExporter(workspace, name);
+    }
+
+    async registerMetadataExporter(
+        workspace: string,
+        name: string,
+        matcher: BlockMatcher,
+        exporter: BlockContentExporter<
+            Array<[string, number | string | string[]]>
+        >
+    ) {
+        await this.database.registerMetadataExporter(
+            workspace,
+            name,
+            matcher,
+            exporter
+        );
+    }
+
+    async unregisterMetadataExporter(workspace: string, name: string) {
+        await this.database.unregisterMetadataExporter(workspace, name);
+    }
+
+    async registerTagExporter(
+        workspace: string,
+        name: string,
+        matcher: BlockMatcher,
+        exporter: BlockContentExporter<string[]>
+    ) {
+        await this.database.registerTagExporter(
+            workspace,
+            name,
+            matcher,
+            exporter
+        );
+    }
+
+    async unregisterTagExporter(workspace: string, name: string) {
+        await this.database.unregisterTagExporter(workspace, name);
+    }
+
+    protected async _observe(
+        workspace: string,
+        blockId: string,
+        callback: ObserveCallback
+    ): Promise<ReturnUnobserve> {
+        return await this.database.observe(workspace, blockId, async states => {
+            const db = await this.database.getDatabase(workspace);
+            const new_block = await db.get(blockId as 'block');
+            callback(states, new_block);
+        });
+    }
+
+    protected async _unobserve(
+        workspace: string,
+        blockId: string,
+        callback?: ObserveCallback
+    ) {
+        return await this.database.unobserve(workspace, blockId, callback);
+    }
+}

libs/datasource/db-service/src/services/comment/comment.ts

@@ -0,0 +1,315 @@
+import { DependencyCallOrConstructProps } from '@toeverything/utils';
+import type { ReturnUnobserve } from '../database/observer';
+import {
+    DeleteEditorBlock,
+    GetEditorBlock,
+    ReturnEditorBlock,
+} from '../editor-block/types';
+import { EditorBlock, ObserveCallback } from '../editor-block';
+import {
+    CommentReply,
+    CreateCommentBlock,
+    CreateReplyBlock,
+    UpdateCommentBlock,
+    UpdateReplyBlock,
+    GetCommentsBlock,
+    Comment,
+} from './types';
+import { DefaultColumnsValue } from './../index';
+import { CommentColumnValue } from '../editor-block/utils/column/types';
+import { WORKSPACE_COMMENTS } from '../../utils';
+
+export class CommentService {
+    protected editor_block: EditorBlock;
+    protected get_dependency: DependencyCallOrConstructProps['getDependency'];
+    constructor(props: DependencyCallOrConstructProps) {
+        this.get_dependency = props.getDependency;
+        this.editor_block = this.get_dependency(EditorBlock);
+    }
+
+    async createComment({
+        workspace,
+        pageId,
+        attachedToBlocksIds,
+        quote,
+        content,
+    }: CreateCommentBlock): Promise<{ commentsId: string } | undefined> {
+        const rootCommentId = await this.get_root_comment_id({
+            workspace,
+            pageId,
+        });
+        const discussionBlock = await this.editor_block.create({
+            workspace: workspace,
+            type: 'comments',
+        });
+
+        if (!discussionBlock) {
+            return undefined;
+        }
+
+        const parentBlock = await this.editor_block.get({
+            workspace,
+            ids: [rootCommentId],
+        });
+
+        if (parentBlock.length === 0) {
+            return undefined;
+        }
+
+        const children = parentBlock[0]?.children || [];
+        children.push(discussionBlock.id);
+        const success = await this.editor_block.update({
+            id: rootCommentId,
+            workspace: workspace,
+            children: children,
+        });
+        if (!success) {
+            return undefined;
+        }
+
+        let result = false;
+        result = await this.updateComment({
+            workspace: workspace,
+            id: discussionBlock.id,
+            attachedToBlocksIds: attachedToBlocksIds || [],
+            quote: quote,
+            pageId: pageId,
+        });
+        if (!result) {
+            return undefined;
+        }
+
+        const reply = await this.createReply({
+            workspace: workspace,
+            parentId: discussionBlock.id,
+            content: content,
+        });
+
+        return { commentsId: discussionBlock.id };
+    }
+
+    async createReply({
+        workspace,
+        parentId,
+        content,
+    }: CreateReplyBlock): Promise<boolean> {
+        const reply_block = await this.editor_block.create({
+            workspace: workspace,
+            type: 'comment',
+        });
+        if (!reply_block) {
+            return false;
+        }
+
+        const parent_block = await this.editor_block.get({
+            workspace,
+            ids: [parentId],
+        });
+
+        if (parent_block.length === 0) {
+            return false;
+        }
+
+        const children = parent_block[0]?.children || [];
+        children.push(reply_block.id);
+        const success = await this.editor_block.update({
+            id: parentId,
+            workspace: workspace,
+            children: children,
+        });
+        if (!success) {
+            return false;
+        }
+
+        return await this.updateReply({
+            workspace,
+            id: reply_block.id,
+            content,
+        });
+    }
+
+    async updateComment({
+        workspace,
+        id,
+        pageId,
+        attachedToBlocksIds,
+        quote,
+        resolve,
+    }: UpdateCommentBlock): Promise<boolean> {
+        const properties: Partial<DefaultColumnsValue> = {};
+        if (quote !== undefined) {
+            properties.text = quote;
+        }
+        const blocks = await this.editor_block.get({
+            workspace,
+            ids: [id],
+        });
+        if (blocks.length === 0) {
+            return false;
+        }
+        const commentValue: CommentColumnValue = {} as CommentColumnValue;
+        Object.assign(commentValue, blocks[0]?.properties?.comment);
+        if (pageId !== undefined) {
+            commentValue.pageId = pageId;
+        }
+        if (attachedToBlocksIds !== undefined) {
+            commentValue.attachedToBlocksIds = attachedToBlocksIds;
+        }
+        if (resolve !== undefined) {
+            commentValue.resolve = resolve;
+            commentValue.finishTime = resolve ? Date.now() : undefined;
+            commentValue.resolveUserId = resolve
+                ? await this.editor_block.getUserId(workspace)
+                : undefined;
+        }
+        properties.comment = commentValue;
+        return await this.editor_block.update({
+            id: id,
+            workspace: workspace,
+            properties: properties,
+        });
+    }
+
+    async updateReply({
+        workspace,
+        id,
+        content,
+    }: UpdateReplyBlock): Promise<boolean> {
+        return await this.editor_block.update({
+            id: id,
+            workspace: workspace,
+            properties: {
+                text: content,
+            },
+        });
+    }
+
+    async delete({ workspace, id }: DeleteEditorBlock): Promise<boolean> {
+        return await this.editor_block.delete({
+            workspace: workspace,
+            id: id,
+        });
+    }
+
+    async getPageComments({
+        workspace,
+        pageId,
+    }: GetCommentsBlock): Promise<Comment | null> {
+        const root_comment_id = await this.get_root_comment_id({
+            workspace,
+            pageId,
+        });
+        const comments = await this.getComments({
+            workspace,
+            ids: [root_comment_id],
+        });
+        return comments.length > 0 ? comments[0] : null;
+    }
+
+    async getComments({
+        workspace,
+        ids,
+    }: GetEditorBlock): Promise<Array<Comment>> {
+        const blocks = (await this.get({ workspace, ids })).filter(block => {
+            return (
+                block &&
+                block?.type === 'comments' &&
+                !block.properties?.comment?.resolve
+            );
+        }) as ReturnEditorBlock[];
+        const comments = blocks.map(block => {
+            return {
+                id: block.id,
+                workspace: block.workspace,
+                type: block.type,
+                parentId: block.parentId,
+                attachedToBlocksIds:
+                    block.properties?.comment?.attachedToBlocksIds || [],
+                children: block.children,
+                quote: block.properties?.text || [],
+                resolve: block.properties?.comment?.resolve || false,
+                resolveUserId: block.properties?.comment?.resolveUserId || '',
+                created: block.created,
+                lastUpdated: block.lastUpdated,
+                creator: block.creator,
+            } as Comment;
+        });
+        return comments;
+    }
+
+    async getReplyList({
+        workspace,
+        ids,
+    }: GetEditorBlock): Promise<Array<CommentReply>> {
+        const blocks = (await this.editor_block.get({ workspace, ids })).filter(
+            block => block?.type === 'comment'
+        ) as ReturnEditorBlock[];
+
+        const replyList = blocks.map(block => {
+            return {
+                id: block.id,
+                workspace: block.workspace,
+                type: block.type,
+                parentId: block.parentId,
+                children: block.children,
+                content: block.properties?.text || [],
+                created: block.created,
+                lastUpdated: block.lastUpdated,
+                creator: block.creator,
+            } as CommentReply;
+        });
+        return replyList;
+    }
+
+    async get({
+        workspace,
+        ids,
+    }: GetEditorBlock): Promise<Array<ReturnEditorBlock | null>> {
+        const blocks = await this.editor_block.get({
+            workspace,
+            ids,
+        });
+        return blocks;
+    }
+
+    async observe(
+        { workspace, id }: DeleteEditorBlock,
+        callback: ObserveCallback
+    ): Promise<ReturnUnobserve> {
+        return await this.editor_block.observe({ workspace, id }, callback);
+    }
+
+    async unobserve({ workspace, id }: DeleteEditorBlock) {
+        await this.editor_block.unobserve({ workspace, id });
+    }
+
+    private async get_root_comment_id({
+        workspace,
+        pageId,
+    }: GetCommentsBlock): Promise<string> {
+        const workspace_db_block = await this.editor_block.getWorkspaceDbBlock(
+            workspace
+        );
+        const workspace_comments: any[] =
+            workspace_db_block.getDecoration(WORKSPACE_COMMENTS) || [];
+        let root_comment = workspace_comments.find(
+            item => item.pageId === pageId
+        );
+        if (!root_comment) {
+            const discussion_block = await this.editor_block.create({
+                workspace: workspace,
+                type: 'comments',
+            });
+            root_comment = {
+                pageId: pageId,
+                rootCommentId: discussion_block.id,
+            };
+            workspace_comments.push(root_comment);
+            workspace_db_block.setDecoration(
+                WORKSPACE_COMMENTS,
+                workspace_comments
+            );
+        }
+        return root_comment.rootCommentId;
+    }
+}

libs/datasource/db-service/src/services/comment/index.ts

@@ -0,0 +1,2 @@
+export * from './comment';
+export * from './types';

libs/datasource/db-service/src/services/comment/types.ts

@@ -0,0 +1,64 @@
+import { ContentColumnValue, BlockFlavorKeys } from '../index';
+export interface CommentReply {
+    id: string;
+    workspace: string;
+    type: BlockFlavorKeys;
+    parentId?: string;
+    children: string[];
+    content: ContentColumnValue;
+    created: number;
+    lastUpdated: number;
+    creator?: string;
+}
+
+export interface Comment {
+    id: string;
+    workspace: string;
+    type: BlockFlavorKeys;
+    parentId?: string;
+    /** store the block-ids where comment is on,
+     * useful when comment is not page level but on a specific block in page  */
+    attachedToBlocksIds?: string[];
+    children: string[];
+    quote: ContentColumnValue;
+    resolve: boolean;
+    resolveUserId?: string;
+    created: number;
+    lastUpdated: number;
+    creator?: string;
+}
+
+export interface CreateCommentBlock {
+    workspace: string;
+    pageId: string;
+    attachedToBlocksIds?: string[];
+    quote: ContentColumnValue;
+    content: ContentColumnValue;
+}
+
+export interface CreateReplyBlock {
+    workspace: string;
+    parentId: string;
+    content: ContentColumnValue;
+}
+
+export interface UpdateCommentBlock {
+    workspace: string;
+    id: string;
+    pageId?: string;
+    attachedToBlocksIds?: string[];
+    quote?: ContentColumnValue;
+    resolve?: boolean;
+}
+
+export interface UpdateReplyBlock {
+    workspace: string;
+    id: string;
+    content: ContentColumnValue;
+    parentId?: string;
+}
+
+export interface GetCommentsBlock {
+    workspace: string;
+    pageId: string;
+}

libs/datasource/db-service/src/services/database/index.ts

@@ -0,0 +1,180 @@
+import { getAuth, type User } from 'firebase/auth';
+
+import {
+    BlockClient,
+    BlockClientInstance,
+    BlockContentExporter,
+    BlockMatcher,
+    BlockInitOptions,
+} from '@toeverything/datasource/jwt';
+import { sleep } from '@toeverything/utils';
+
+import { ObserverManager, getObserverName } from './observer';
+import type { ObserveCallback, ReturnUnobserve } from './observer';
+export type { ObserveCallback, ReturnUnobserve } from './observer';
+
+const workspaces: Record<string, BlockClientInstance> = {};
+
+const loading = new Set();
+
+const waitLoading = async (key: string) => {
+    while (loading.has(key)) {
+        await sleep();
+    }
+};
+
+async function _getCurrentToken() {
+    if (process.env['NX_FREE_LOGIN']) {
+        return 'NX_FREE_LOGIN';
+    }
+    const token = await getAuth().currentUser?.getIdToken();
+    if (token) return token;
+    return new Promise<string>(resolve => {
+        getAuth().onIdTokenChanged((user: User | null) => {
+            if (user) resolve(user.getIdToken());
+        });
+    });
+}
+
+async function _getBlockDatabase(
+    workspace: string,
+    options?: BlockInitOptions
+) {
+    if (loading.has(workspace)) {
+        await waitLoading(workspace);
+    }
+
+    // if (
+    //     options?.userId &&
+    //     workspaces[workspace]?.getUserId() !== options?.userId
+    // ) {
+    //     delete workspaces[workspace];
+    // }
+
+    if (!workspaces[workspace]) {
+        loading.add(workspace);
+
+        workspaces[workspace] = await BlockClient.init(workspace, {
+            ...options,
+            token: await _getCurrentToken(),
+        });
+        (window as any).client = workspaces[workspace];
+        await workspaces[workspace].buildIndex();
+        loading.delete(workspace);
+    }
+    return workspaces[workspace];
+}
+
+interface DatabaseProps {
+    options?: BlockInitOptions;
+}
+
+export class Database {
+    readonly #observers = new ObserverManager();
+
+    readonly #options?: BlockInitOptions;
+    constructor(props: DatabaseProps) {
+        this.#options = props.options;
+    }
+
+    async getDatabase(workspace: string, options?: BlockInitOptions) {
+        const db = await _getBlockDatabase(workspace, options);
+        return db;
+    }
+
+    async registerContentExporter(
+        workspace: string,
+        name: string,
+        matcher: BlockMatcher,
+        exporter: BlockContentExporter
+    ) {
+        const db = await this.getDatabase(workspace);
+        db.registerContentExporter(name, matcher, exporter);
+    }
+
+    async unregisterContentExporter(workspace: string, name: string) {
+        const db = await this.getDatabase(workspace);
+        db.unregisterContentExporter(name);
+    }
+
+    async registerMetadataExporter(
+        workspace: string,
+        name: string,
+        matcher: BlockMatcher,
+        exporter: BlockContentExporter<
+            Array<[string, number | string | string[]]>
+        >
+    ) {
+        const db = await this.getDatabase(workspace);
+        db.registerMetadataExporter(name, matcher, exporter);
+    }
+
+    async unregisterMetadataExporter(workspace: string, name: string) {
+        const db = await this.getDatabase(workspace);
+        db.unregisterMetadataExporter(name);
+    }
+
+    async registerTagExporter(
+        workspace: string,
+        name: string,
+        matcher: BlockMatcher,
+        exporter: BlockContentExporter<string[]>
+    ) {
+        const db = await this.getDatabase(workspace);
+        db.registerTagExporter(name, matcher, exporter);
+    }
+
+    async unregisterTagExporter(workspace: string, name: string) {
+        const db = await this.getDatabase(workspace);
+        db.unregisterTagExporter(name);
+    }
+
+    async observe(
+        workspace: string,
+        blockId: string,
+        callback: ObserveCallback
+    ): Promise<ReturnUnobserve> {
+        const observer_name = getObserverName(workspace, blockId);
+        const unobserve = this.#observers.addCallback(observer_name, callback);
+        if (this.#observers.getStatus(observer_name) === 'observing') {
+            return unobserve;
+        }
+        const db = await this.getDatabase(workspace, this.#options);
+        const block = await db.get(blockId as 'block');
+        if (block) {
+            const listener: Parameters<
+                typeof block['on']
+            >[2] = async states => {
+                const new_block = await db.get(blockId as 'block');
+                this.#observers.getCallbacks(observer_name).forEach(cb => {
+                    cb(states, new_block);
+                });
+            };
+
+            block.on('children', observer_name, listener);
+            block.on('content', observer_name, listener);
+            block.on('parent', observer_name, listener);
+        }
+        this.#observers.setStatus(observer_name, 'observing');
+
+        return unobserve;
+    }
+
+    async unobserve(
+        workspace: string,
+        blockId: string,
+        callback?: ObserveCallback
+    ) {
+        const observer_name = getObserverName(workspace, blockId);
+        this.#observers.removeCallback(observer_name, callback);
+        if (!this.#observers.getCallbacks(observer_name).length) {
+            const db = await this.getDatabase(workspace, this.#options);
+            const block = await db.get(blockId as 'block');
+            if (block) {
+                block.off('children', observer_name);
+                block.off('content', observer_name);
+                block.off('parent', observer_name);
+            }
+        }
+    }
+}

libs/datasource/db-service/src/services/database/observer.ts

@@ -0,0 +1,69 @@
+import { ChangedStates, BlockImplInstance } from '@toeverything/datasource/jwt';
+
+export type ObserveCallback = (
+    changeStates: ChangedStates,
+    block: BlockImplInstance
+) => void;
+export type ReturnUnobserve = () => void;
+
+type ObserverStatus = {
+    status: 'observing' | 'removing' | 'none';
+    callbacks: ObserveCallback[];
+};
+
+export class ObserverManager {
+    private observe_callbacks: Record<string, ObserverStatus | undefined> = {};
+    addCallback(key: string, callback: ObserveCallback) {
+        if (!this.observe_callbacks[key]) {
+            this.observe_callbacks[key] = {
+                status: 'none',
+                callbacks: [],
+            };
+        }
+
+        const observer = this.observe_callbacks[key] as ObserverStatus;
+        observer.callbacks.push(callback);
+        return () => {
+            const index = observer.callbacks.indexOf(callback);
+            if (index > -1) {
+                observer.callbacks.splice(index, 1);
+            }
+        };
+    }
+    removeCallback(key: string, callback?: ObserveCallback) {
+        const observer = this.observe_callbacks[key];
+        if (!observer) {
+            return;
+        }
+        if (callback) {
+            const index = observer.callbacks.indexOf(callback);
+            if (index > -1) {
+                observer.callbacks.splice(index, 1);
+            }
+        } else {
+            observer.callbacks = [];
+        }
+    }
+    getCallbacks(key: string) {
+        return this.observe_callbacks[key]?.callbacks || [];
+    }
+    getStatus(key: string) {
+        return this.observe_callbacks[key]?.status || 'none';
+    }
+    setStatus(key: string, status: 'observing' | 'removing' | 'none') {
+        if (!this.observe_callbacks[key]) {
+            this.observe_callbacks[key] = {
+                status: 'none',
+                callbacks: [],
+            };
+        }
+        (this.observe_callbacks[key] as ObserverStatus).status = status;
+    }
+    removeObserve(key: string) {
+        this.observe_callbacks[key] = undefined;
+    }
+}
+
+export function getObserverName(workspace: string, blockId: string) {
+    return `${workspace}_${blockId}`;
+}

libs/datasource/db-service/src/services/editor-block/index.ts

@@ -0,0 +1,431 @@
+import { diffArrays } from 'diff';
+import { has } from '@toeverything/utils';
+import { ServiceBaseClass } from '../base';
+import type { ReturnUnobserve } from '../database/observer';
+import {
+    CreateEditorBlock,
+    ReturnEditorBlock,
+    GetEditorBlock,
+    UpdateEditorBlock,
+    DeleteEditorBlock,
+    AddColumnProps,
+    RemoveColumnProps,
+    UpdateColumnProps,
+    BlockFlavorKeys,
+} from './types';
+import {
+    dbBlock2BusinessBlock,
+    serializeColumnConfig,
+    deserializeColumnConfig,
+    getOrInitBlockContentColumnsField,
+    addColumn,
+    Column,
+} from './utils';
+import { BlockImplInstance, MapOperation } from '@toeverything/datasource/jwt';
+import { TemplateProperties, Template } from './templates/types';
+export type ObserveCallback = (businessBlock: ReturnEditorBlock) => void;
+export class EditorBlock extends ServiceBaseClass {
+    async create({
+        workspace,
+        type,
+        parentId,
+    }: CreateEditorBlock): Promise<ReturnEditorBlock> {
+        const db = await this.database.getDatabase(workspace);
+        const dbBlock = await db.get(type as 'block');
+        if (parentId) {
+            const parentBlock = await db.get(parentId as 'block');
+            if (parentBlock.id === parentId) {
+                parentBlock.insertChildren(dbBlock);
+            }
+        }
+        // Initialize the columns field of the block
+        getOrInitBlockContentColumnsField(dbBlock);
+        return dbBlock2BusinessBlock({
+            workspace,
+            dbBlock,
+        }) as ReturnEditorBlock;
+    }
+
+    async get({
+        workspace,
+        ids,
+    }: GetEditorBlock): Promise<Array<ReturnEditorBlock | null>> {
+        const blocks = await Promise.all(
+            ids.map(async id => {
+                const block = await this.getBlock(workspace, id);
+                return dbBlock2BusinessBlock({
+                    workspace,
+                    dbBlock: block,
+                });
+            })
+        );
+        return blocks;
+    }
+
+    async getBlockByFlavor(
+        workspace: string,
+        flavor: BlockFlavorKeys
+    ): Promise<string[]> {
+        const db = await this.database.getDatabase(workspace);
+        const keys: string[] = await db.getBlockByFlavor(flavor);
+        return keys;
+    }
+
+    async getUserId(workspace: string): Promise<string> {
+        const db = await this.database.getDatabase(workspace);
+        return db.getUserId();
+    }
+
+    async update(businessBlock: UpdateEditorBlock): Promise<boolean> {
+        const db = await this.database.getDatabase(businessBlock.workspace);
+        if (!businessBlock.id) {
+            return false;
+        }
+        const db_block = await this.getBlock(
+            businessBlock.workspace,
+            businessBlock.id as 'block'
+        );
+        if (!db_block) {
+            return false;
+        }
+        if (
+            has(businessBlock, 'type') &&
+            businessBlock.type !== db_block.flavor
+        ) {
+            db_block.setFlavor(businessBlock.type as 'text');
+        }
+        if (
+            has(businessBlock, 'parentId') &&
+            businessBlock.parentId !== db_block.parent?.id
+        ) {
+            db_block.remove();
+            const parent = await db.get(businessBlock.id as 'block');
+            if (!parent) {
+                return false;
+            }
+            parent.append(db_block);
+        }
+        if (
+            has(businessBlock, 'children') &&
+            businessBlock.children !== db_block.children
+        ) {
+            const patches = diffArrays(
+                db_block.children || [],
+                businessBlock.children || []
+            );
+            let position = 0;
+            for (let i = 0; i < patches.length; i++) {
+                const patch = patches[i];
+                if (patch.added) {
+                    if (patch.value.length) {
+                        for (let i = 0; i < patch.value.length; i++) {
+                            const child = await db.get(
+                                patch.value[i] as 'block'
+                            );
+                            if (child && child.id === patch.value[i]) {
+                                db_block.insertChildren(child, {
+                                    pos: position,
+                                });
+                                position = position + 1;
+                            }
+                        }
+                    }
+                } else if (patch.removed) {
+                    patch.value.forEach(child_id => {
+                        db_block.removeChildren(child_id);
+                    });
+                } else if (patch.count) {
+                    position = position + patch.count;
+                }
+            }
+        }
+
+        const decorations = db_block.getDecorations();
+        Object.entries(businessBlock.properties || {}).forEach(
+            ([key, value]) => {
+                if (decorations[key] !== value) {
+                    db_block.setDecoration(key, value);
+                }
+            }
+        );
+        return true;
+    }
+
+    async delete({ workspace, id }: DeleteEditorBlock): Promise<boolean> {
+        const db = await this.database.getDatabase(workspace);
+        const db_block = await db.get(id as 'block');
+        if (!db_block) {
+            return false;
+        }
+        db_block.remove();
+        return true;
+    }
+    async suspend(workspace: string, flag: boolean): Promise<void> {
+        const db = await this.database.getDatabase(workspace);
+        db.suspend(flag);
+    }
+
+    async addColumn({
+        workspace,
+        blockId,
+        column,
+    }: AddColumnProps): Promise<boolean> {
+        const db_block = await this.getBlock(workspace, blockId);
+        if (!db_block) {
+            return false;
+        }
+        const columns = getOrInitBlockContentColumnsField(db_block);
+        if (!columns) {
+            return false;
+        }
+        addColumn({
+            block: db_block,
+            columns,
+            columnConfig: column,
+        });
+        return true;
+    }
+
+    async updateColumn({
+        workspace,
+        blockId,
+        columnId,
+        column,
+    }: UpdateColumnProps): Promise<boolean> {
+        const db_block = await this.getBlock(workspace, blockId);
+        if (!db_block) {
+            return false;
+        }
+        const columns = getOrInitBlockContentColumnsField(db_block);
+        if (!columns) {
+            return false;
+        }
+        const old_column = columns.find<MapOperation<string>>(col => {
+            // @ts-ignore TODO: don't know why
+            return col.get('id') === columnId;
+        });
+        if (!old_column) {
+            return false;
+        }
+        const column_config = {
+            // @ts-ignore TODO: don't know why
+            ...deserializeColumnConfig(old_column?.get('config') as string),
+            ...column,
+        };
+        old_column?.set(
+            'config',
+            // @ts-ignore TODO: don't know why
+            serializeColumnConfig(column_config as Column)
+        );
+        return true;
+    }
+
+    async removeColumn({
+        workspace,
+        blockId,
+        columnId,
+    }: RemoveColumnProps): Promise<boolean> {
+        const db_block = await this.getBlock(workspace, blockId);
+        if (!db_block) {
+            return false;
+        }
+        const columns = getOrInitBlockContentColumnsField(db_block);
+        if (columns?.length) {
+            // @ts-ignore TODO: don't know why
+            const idx = columns?.findIndex(col => col.get('id') === columnId);
+            if (idx > -1) {
+                columns.delete(idx, 1);
+            }
+        }
+        return true;
+    }
+    private async decorate_page_title(
+        page_block: BlockImplInstance,
+        prefix: string
+    ) {
+        const text = page_block.getDecoration('text');
+        if (page_block && text) {
+            const new_text = JSON.parse(JSON.stringify(text));
+            //@ts-ignore
+            new_text.value[0].text = prefix + new_text.value[0].text;
+            page_block.setDecoration('text', new_text);
+        }
+    }
+    private async update_page_title(
+        pageBlock: BlockImplInstance,
+        title: string
+    ) {
+        if (title) {
+            pageBlock.setDecoration('text', { value: [{ text: title }] });
+        }
+    }
+    async copyPage(
+        workspace_id: string,
+        source_page_id: string,
+        new_page_id: string
+    ): Promise<boolean> {
+        const db = await this.database.getDatabase(workspace_id);
+
+        const source_page = await this.getBlock(
+            workspace_id,
+            source_page_id as 'block'
+        );
+        const new_page = await this.getBlock(
+            workspace_id,
+            new_page_id as 'block'
+        );
+        if (!source_page) {
+            return false;
+        }
+        const source_page_children = source_page.children;
+        const decorations = source_page.getDecorations();
+        Object.entries(decorations).forEach(([key, value]) => {
+            new_page?.setDecoration(key, source_page.getDecoration(key));
+        });
+
+        //@ts-ignore
+        this.decorate_page_title(new_page, 'copy from ');
+
+        for (let i = 0; i < source_page_children.length; i++) {
+            const source_page_child = await db.get(
+                source_page_children[i] as 'block'
+            );
+            new_page?.insertChildren(source_page_child);
+        }
+        return true;
+    }
+    async copyTemplateToPage(
+        workspace: string,
+        sourcePageId: string,
+        templateData: Template
+    ) {
+        const db = await this.database.getDatabase(workspace);
+        const sourcePage = await this.getBlock(
+            workspace,
+            sourcePageId as 'block'
+        );
+
+        if (!sourcePage) {
+            return false;
+        }
+        if (templateData.properties && templateData.properties.text) {
+            this.update_page_title(
+                sourcePage,
+                templateData.properties.text?.value[0].text
+            );
+        }
+
+        this.update_block_properies(sourcePage, templateData.properties);
+        if (!templateData.blocks) return false;
+        for (let i = 0; i < templateData.blocks.length; i++) {
+            const blockData = templateData.blocks[i];
+            const sourcPageChild = await db.get(blockData.type as 'block');
+
+            this.update_block_properies(sourcPageChild, blockData.properties);
+
+            sourcePage?.insertChildren(sourcPageChild);
+
+            await this.copyTemplateToBlocks(
+                workspace,
+                sourcPageChild.id,
+                blockData
+            );
+        }
+        return true;
+    }
+    private update_block_properies(
+        block: BlockImplInstance,
+        properties: TemplateProperties
+    ) {
+        Object.entries(properties).forEach(([key, value]) => {
+            block.setDecoration(key, value);
+        });
+    }
+    async copyTemplateToBlocks(
+        workspace: string,
+        parentBlockId: string,
+        template: Template
+    ) {
+        const db = await this.database.getDatabase(workspace);
+        const parentBlock = await this.getBlock(
+            workspace,
+            parentBlockId as 'block'
+        );
+
+        if (!parentBlock) {
+            return false;
+        }
+        if (!template.blocks) return true;
+        for (let i = 0; i < template.blocks.length; i++) {
+            const blockData = template.blocks[i];
+            const sourcPageChild = await db.get(blockData.type as 'block');
+
+            this.update_block_properies(sourcPageChild, blockData.properties);
+
+            parentBlock?.insertChildren(sourcPageChild);
+            if (blockData.blocks) {
+                this.copyTemplateToBlocks(
+                    workspace,
+                    sourcPageChild.id,
+                    blockData
+                );
+            }
+        }
+        return true;
+    }
+    async observe(
+        { workspace, id }: DeleteEditorBlock,
+        callback: ObserveCallback
+    ): Promise<ReturnUnobserve> {
+        return await this._observe(workspace, id, async (states, block) => {
+            callback(
+                dbBlock2BusinessBlock({
+                    workspace,
+                    dbBlock: block,
+                }) as ReturnEditorBlock
+            );
+        });
+    }
+
+    async unobserve({ workspace, id }: DeleteEditorBlock) {
+        await this._unobserve(workspace, id);
+    }
+}
+
+export type {
+    CreateEditorBlock,
+    ReturnEditorBlock,
+    GetEditorBlock,
+    UpdateEditorBlock,
+    DeleteEditorBlock,
+    BlockFlavors,
+    BlockFlavorKeys,
+} from './types';
+
+export type {
+    Column,
+    ContentColumn,
+    NumberColumn,
+    EnumColumn,
+    DateColumn,
+    BooleanColumn,
+    FileColumn,
+    DefaultColumnsValue,
+    ContentColumnValue,
+    NumberColumnValue,
+    EnumColumnValue,
+    BooleanColumnValue,
+    DateColumnValue,
+    FileColumnValue,
+    StringColumnValue,
+} from './utils/column';
+export {
+    ColumnType,
+    isBooleanColumn,
+    isContentColumn,
+    isDateColumn,
+    isFileColumn,
+    isNumberColumn,
+    isEnumColumn,
+    isStringColumn,
+} from './utils/column';

libs/datasource/db-service/src/services/editor-block/templates/group-templates.ts

@@ -0,0 +1,604 @@
+//@ts-nocheck
+import { GroupTemplate } from './types';
+type GroupTemplateMap = Record<GroupTemplateKeys, GroupTemplate>;
+const groupTemplateMap: GroupTemplateMap = {
+    empty: {
+        type: 'group',
+        properties: {},
+        blocks: [
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [{ text: '' }],
+                    },
+                },
+                blocks: [],
+            },
+        ],
+    },
+    todolist: {
+        type: 'group',
+        properties: {},
+        blocks: [
+            {
+                type: 'heading1',
+                properties: {
+                    text: {
+                        value: [{ text: '🎓Graduating from the project' }],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Congratulations! Now you can start create your own projects!',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'To start using workspaces and folders hit the ',
+                            },
+                            {
+                                bold: true,
+                                text: 'button at the top left of the screen',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: '',
+                            },
+                            {
+                                text: 'At any time if you feel lost do visit our 📚Blog: ',
+                            },
+                            {
+                                type: 'link',
+                                url: 'https://blog.affine.pro/',
+                                id: 'link.qx4yhw81or54',
+                                children: [
+                                    {
+                                        text: 'https://blog.affine.pro',
+                                    },
+                                ],
+                            },
+                            {
+                                text: '',
+                            },
+                        ],
+                    },
+                    collapsed: {
+                        value: false,
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: '',
+                            },
+                            {
+                                text: 'If you have any suggestions drop a post in our Reddit Channel：',
+                            },
+                            {
+                                type: 'link',
+                                url: 'https://www.reddit.com/r/Affine/',
+                                id: 'link.zeafc4ogfvrb',
+                                children: [
+                                    {
+                                        text: 'https://www.reddit.com/r/Affine/',
+                                    },
+                                ],
+                            },
+                            {
+                                text: ' ',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading2',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: '🎉 The Essentials. Check things off after you tried them!',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            { text: ' ✅ ' },
+                            { bold: true, text: 'Check' },
+                            {
+                                text: ' the text box here to complete the task!',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            { text: '' },
+                            { text: '' },
+                            { text: ' 👋 ' },
+                            { bold: true, text: 'Drag' },
+                            {
+                                text: ' the ⠟ button left of the checkbox to reorder tasks',
+                            },
+                            { text: '' },
+                            { text: '' },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'todo',
+                properties: {
+                    text: {
+                        value: [
+                            { text: '' },
+                            { text: '' },
+                            { text: ' ➡️ ' },
+                            { bold: true, text: 'Fold' },
+                            { text: ' and ' },
+                            { bold: true, text: 'Unfold' },
+                            {
+                                text: ' a task to simplify your list using the arrow on the right ⤵️',
+                            },
+                        ],
+                    },
+                    numberType: 'type1',
+                    collapsed: { value: false },
+                },
+                blocks: [
+                    {
+                        type: 'figma',
+                        properties: {
+                            embedLink: {
+                                value: 'https://www.figma.com/embed?embed_host=share&url=https%3A%2F%2Fwww.figma.com%2Ffile%2F7pyx5gMz6CN0qSRADmScQ7%2FAFFINE%3Fnode-id%3D40%253A2',
+                                name: 'figma',
+                            },
+                        },
+                        blocks: [],
+                    },
+                ],
+            },
+        ],
+    },
+    blog: {
+        type: 'group',
+        properties: {},
+        blocks: [
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'As a collaborative real-time editor, Affine aims to resolve problems in three situations:',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'bullet',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Multi-master replication: the synchronization of data between equipment and applications;',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'bullet',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Eventual consistency: the consistence of data regardless of network latency and outage;',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'bullet',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Conflict resolution: the resolution of conflict between simultaneous edits.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'To achieve these aims, a proper collaborative algorithm should be used. There are hundreds of collaborative algorithms being invented over the past three decades, but they usually fall into two categories: either operational transformation (OT) or conflict-free replicated data type (CRDT). We think CRDT is a better choice.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading1',
+                properties: {
+                    text: {
+                        value: [{ bold: true, text: 'What does CRDT do' }],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'CRDT is capable of discovering and resolving conflicts while ensuring the effective distribution and merge of date. It may sound like magic, but think about historical study, it is very possible to discover the truth from scattered evidence.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'For CRDT, every piece of data is like a "historic fragment". We keep collecting the fragments from other clients and then restore the truth by excluding repeated data and correcting false information.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading1',
+                properties: {
+                    text: {
+                        value: [{ bold: true, text: 'Why CRDT is better' }],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'In contrast to OT, CRDT possesses three big advantages:',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading2',
+                properties: {
+                    text: {
+                        value: [{ bold: true, text: 'Flexibility' }],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'CRDT supports more data types. For example, Yjs supports Array, Map, and Treelike, and therefore applies to more business scenarios.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading2',
+                properties: {
+                    text: {
+                        value: [{ bold: true, text: 'Performance' }],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'CRDT tolerates higher latency and can wait longer for solving conflicts, whereas the calculation of OT in the same condition may become too overwhelming for a server to sustain.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading2',
+                properties: {
+                    text: {
+                        value: [{ bold: true, text: 'Extensibility' }],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Because CRDT supports more data types and editor elements, it is more extensible.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading1',
+                properties: {
+                    text: { value: [{ text: 'Conclusion' }] },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Collaborative algorithm is still a foreign concept for many developers. There are some introductions to it, but as to how it shall be used, there is still lack of clear explanation. I hope this article helps. If you also work for a startup company and want some suggestions, CRDT, especially Yjs, should be a better choice.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: { text: { value: [{ text: '' }] } },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            { text: '' },
+                            {
+                                type: 'link',
+                                url: 'https://blog.affine.pro/',
+                                id: 'link.stubssslo0rq',
+                                children: [{ text: '← View all posts' }],
+                            },
+                            { text: '' },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+        ],
+    },
+    grid: {
+        type: 'group',
+        properties: {},
+        blocks: [
+            {
+                type: 'heading2',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                bold: true,
+                                text: 'Performance',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'CRDT tolerates higher latency and can wait longer for solving conflicts, whereas the calculation of OT in the same condition may become too overwhelming for a server to sustain.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading2',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                bold: true,
+                                text: 'Extensibility',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Because CRDT supports more data types and editor elements, it is more extensible.',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'heading1',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: 'Conclusion',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'grid',
+                properties: {},
+                blocks: [
+                    {
+                        type: 'gridItem',
+                        properties: {
+                            gridItemWidth: '50%',
+                        },
+                        blocks: [
+                            {
+                                type: 'text',
+                                properties: {
+                                    text: {
+                                        value: [
+                                            {
+                                                text: 'Collaborative algorithm is still a foreign concept for many developers. There are some introductions to it, but as to how it shall be used, there is still lack of clear explanation. I hope this article helps. If you also work for a startup company and want some suggestions, CRDT, especially Yjs, should be a better choice.',
+                                            },
+                                        ],
+                                    },
+                                },
+                                blocks: [],
+                            },
+                        ],
+                    },
+                    {
+                        type: 'gridItem',
+                        properties: {
+                            gridItemWidth: '50%',
+                        },
+                        blocks: [
+                            {
+                                type: 'text',
+                                properties: {
+                                    text: {
+                                        value: [
+                                            {
+                                                text: 'Collaborative algorithm is still a foreign concept for many developers. There are some introductions to it, but as to how it shall be used, there is still lack of clear explanation. I hope this article helps. If you also work for a startup company and want some suggestions, CRDT, especially Yjs, should be a better choice.',
+                                            },
+                                        ],
+                                    },
+                                },
+                                blocks: [],
+                            },
+                        ],
+                    },
+                ],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: '',
+                            },
+                        ],
+                    },
+                },
+                blocks: [],
+            },
+            {
+                type: 'text',
+                properties: {
+                    text: {
+                        value: [
+                            {
+                                text: '',
+                            },
+                            {
+                                type: 'link',
+                                url: 'https://blog.affine.pro/',
+                                id: 'link.stubssslo0rq',
+                                children: [
+                                    {
+                                        text: '← View all posts',
+                                    },
+                                ],
+                            },
+                            {
+                                text: '',
+                            },
+                        ],
+                    },
+                },
+            },
+        ],
+    },
+};
+
+export type GroupTemplateKeys = 'todolist' | 'blog' | 'empty' | 'grid';
+export { groupTemplateMap };

libs/datasource/db-service/src/services/editor-block/templates/index.ts

@@ -0,0 +1,3 @@
+export { TemplateFactory } from './template-factory';
+
+export * from './types';

libs/datasource/db-service/src/services/editor-block/templates/template-factory.ts

@@ -0,0 +1,40 @@
+import { groupTemplateMap } from './group-templates';
+import { Template, TemplateMeta } from './types';
+const defaultTemplateList: Array<TemplateMeta> = [
+    {
+        name: 'New From Quick Start',
+        groupKeys: ['todolist'],
+    },
+    { name: 'New From Grid System', groupKeys: ['grid'] },
+    { name: 'New From Blog', groupKeys: ['blog'] },
+    { name: ' New Todolist', groupKeys: ['todolist'] },
+    { name: ' New Empty Page', groupKeys: ['empty'] },
+];
+const TemplateFactory = {
+    defaultTemplateList: defaultTemplateList,
+    generatePageTemplateByGroupKeys(props: TemplateMeta): Template {
+        const newTitle = props.name || 'Get Started with Affine';
+        const keys = props.groupKeys || [];
+        const blankPage: Template = {
+            type: 'page',
+            properties: {
+                text: { value: [{ text: newTitle }] },
+                fullWidthChecked: false,
+            },
+            blocks: [],
+        };
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            if (key in groupTemplateMap) {
+                blankPage.blocks = blankPage.blocks || [];
+                if (groupTemplateMap[key]) {
+                    blankPage.blocks.push(groupTemplateMap[key]);
+                }
+            }
+        }
+
+        return blankPage;
+    },
+};
+
+export { TemplateFactory };

libs/datasource/db-service/src/services/editor-block/templates/types.ts

@@ -0,0 +1,25 @@
+import { DefaultColumnsValue, BlockFlavorKeys } from './../index';
+import { groupTemplateMap, GroupTemplateKeys } from './group-templates';
+
+// interface Block {
+//     type: BlockFlavorKeys;
+//     properties: Partial<DefaultColumnsValue>;
+// }
+export type TemplateProperties = Partial<DefaultColumnsValue>;
+export interface Template {
+    type: BlockFlavorKeys;
+    properties: TemplateProperties;
+    blocks?: Template[];
+}
+
+export interface GroupTemplate {
+    type: BlockFlavorKeys;
+    properties: TemplateProperties;
+    blocks?: Template[];
+}
+export interface TemplateMeta {
+    name: string | null;
+    groupKeys: Array<GroupTemplateKeys> | [];
+}
+
+// export { Template, TemplateProperties };

libs/datasource/db-service/src/services/editor-block/types.ts

@@ -0,0 +1,75 @@
+import { Protocol } from '../../protocol';
+import { Column, DefaultColumnsValue } from './utils/column';
+
+export type BlockFlavors = typeof Protocol.Block.Type;
+export type BlockFlavorKeys = keyof typeof Protocol.Block.Type;
+
+export interface CreateEditorBlock {
+    workspace: string;
+    type: keyof BlockFlavors;
+    parentId?: string;
+}
+
+export interface ReturnEditorBlock {
+    id: string;
+    workspace: string;
+    type: BlockFlavorKeys;
+    parentId?: string;
+    pageId?: string;
+    closestGroupId?: string;
+    columns?: Column[];
+    children: string[];
+    properties?: Partial<DefaultColumnsValue>;
+    created: number;
+    lastUpdated: number;
+    creator?: string;
+}
+
+export interface GetEditorBlock {
+    ids: string[];
+    workspace: string;
+}
+
+export interface UpdateEditorBlock
+    extends Partial<
+        Pick<ReturnEditorBlock, 'type' | 'parentId' | 'children' | 'properties'>
+    > {
+    id: string;
+    workspace: string;
+}
+
+export interface DeleteEditorBlock {
+    id: string;
+    workspace: string;
+}
+
+export interface AddColumnProps {
+    workspace: string;
+    /**
+     * block id
+     * Support group, page block setting columns
+     */
+    blockId: string;
+    column: Column;
+}
+
+export interface UpdateColumnProps {
+    workspace: string;
+    /**
+     * block id
+     * Support group, page block setting columns
+     */
+    blockId: string;
+    columnId: string;
+    column: Partial<Column>;
+}
+
+export interface RemoveColumnProps {
+    workspace: string;
+    /**
+     * block id
+     * Support group, page block setting columns
+     */
+    blockId: string;
+    columnId: string;
+}

libs/datasource/db-service/src/services/editor-block/utils/column/default-config.ts

@@ -0,0 +1,192 @@
+import {
+    Column,
+    ColumnType,
+    ContentColumnValue,
+    BooleanColumnValue,
+    StringColumnValue,
+    FileColumnValue,
+    DateColumnValue,
+    EnumColumnValue,
+    CommentColumnValue,
+    FilterConstraint,
+    SorterConstraint,
+} from './types';
+
+export enum GroupScene {
+    page = 'page',
+    table = 'table',
+    kanban = 'kanban',
+    whiteboard = 'whiteboard',
+}
+
+/**
+ * @deprecated
+ */
+export enum BlockStatus {
+    notStart = 'notStart',
+    progress = 'progress',
+    done = 'done',
+}
+
+/**
+ * @deprecated
+ */
+export type DefaultColumnsValue = {
+    scene: string;
+    visibleColumnKeys: EnumColumnValue;
+    shapeProps: StringColumnValue;
+    text: ContentColumnValue;
+    textStyle: Record<'textAlign', string>;
+    checked: BooleanColumnValue;
+    collapsed: BooleanColumnValue;
+    embedLink: StringColumnValue;
+    image: FileColumnValue;
+    file: FileColumnValue;
+    endDate: DateColumnValue;
+    status: EnumColumnValue<BlockStatus>;
+    gridItemWidth: string;
+    reference: string;
+    numberType: any;
+    image_style: any;
+    lang: any;
+    fullWidthChecked: boolean;
+    comment: CommentColumnValue;
+    filterConstraint: FilterConstraint;
+    filterWeakSqlConstraint: string;
+    sorterConstraint: SorterConstraint;
+};
+
+export const DEFAULT_COLUMN_KEYS = {
+    Text: 'text',
+    Checked: 'checked',
+} as const;
+
+/**
+ * @deprecated
+ */
+export const DEFAULT_COLUMNS: Column[] = [
+    /** System internal variables */ {
+        // Display mode of group / page
+        name: 'Scene',
+        type: ColumnType.enum,
+        key: 'scene',
+        multiple: 1,
+        options: [
+            { id: 'todo', name: 'Todo List', value: GroupScene.page },
+            { id: 'page', name: 'Table', value: GroupScene.table },
+        ],
+        innerColumn: true,
+    },
+    {
+        // block selected displayed columns
+        name: 'visibleColumnKeys',
+        type: ColumnType.enum,
+        key: 'scene',
+        multiple: 1,
+        /**
+         * All columns that the user can see, empty means unlimited options
+         */
+        options: [],
+        innerColumn: true,
+    },
+    {
+        name: 'collapsed',
+        type: ColumnType.boolean,
+        key: 'collapsed',
+        innerColumn: true,
+    },
+    {
+        name: 'shapeProps',
+        type: ColumnType.string,
+        key: 'shapeProps',
+        mode: 'text',
+        innerColumn: true,
+    },
+    {
+        // text content
+        name: 'Content',
+        type: ColumnType.content,
+        key: DEFAULT_COLUMN_KEYS.Text,
+    },
+    {
+        name: 'Status',
+        type: ColumnType.enum,
+        key: 'status',
+        options: [
+            {
+                id: 'notStart',
+                name: 'Not Start',
+                value: BlockStatus.notStart,
+                color: '#E53535',
+                background: '#FFCECE',
+            },
+            {
+                id: 'progress',
+                name: 'Progress',
+                value: BlockStatus.progress,
+                color: '#A77F1A',
+                background: '#FFF5AB',
+            },
+            {
+                id: 'done',
+                name: 'Done',
+                value: BlockStatus.done,
+                color: '#3C8867',
+                background: '#C5FBE0',
+            },
+        ],
+        multiple: 1,
+        innerColumn: true,
+    },
+    {
+        name: 'Checked',
+        type: ColumnType.boolean,
+        key: DEFAULT_COLUMN_KEYS.Checked,
+        options: [
+            { id: 'checked', name: 'checked', value: true },
+            { id: 'unChecked', name: 'unChecked', value: false },
+        ],
+        sorter: true,
+    },
+    {
+        name: 'Embed Link',
+        type: ColumnType.string,
+        key: 'embedLink',
+        mode: 'url',
+        supportAsTag: true,
+        innerColumn: true,
+    },
+    {
+        name: 'Image',
+        type: ColumnType.file,
+        key: 'image',
+        accept: 'image/gif,image/jpeg,image/jpg,image/png,image/svg',
+        multiple: 1,
+        supportAsTag: true,
+        innerColumn: true,
+    },
+    {
+        name: 'File',
+        type: ColumnType.file,
+        key: 'file',
+        multiple: 1,
+        supportAsTag: true,
+        innerColumn: true,
+    },
+    {
+        name: 'Start Date',
+        type: ColumnType.date,
+        key: 'startDate',
+        format: 'YYYY-MM-dd HH:mm:ss',
+        supportAsTag: true,
+        innerColumn: true,
+    },
+    {
+        name: 'End Date',
+        type: ColumnType.date,
+        key: 'endDate',
+        format: 'YYYY-MM-dd HH:mm:ss',
+        supportAsTag: true,
+        innerColumn: true,
+    },
+];

libs/datasource/db-service/src/services/editor-block/utils/column/index.ts

@@ -0,0 +1,117 @@
+import { nanoid } from 'nanoid';
+import {
+    BlockImplInstance,
+    MapOperation,
+    ArrayOperation,
+} from '@toeverything/datasource/jwt';
+import type { Column } from './types';
+import { DEFAULT_COLUMNS } from './default-config';
+export const serializeColumnConfig = (column: Column): string => {
+    // TODO: Do the type check of the column parameter here
+    return JSON.stringify(column);
+};
+
+export const deserializeColumnConfig = (config: string): Column => {
+    // TODO: do the column check here
+    return JSON.parse(config) as Column;
+};
+
+/**
+ * Support for adding column blocks
+ */
+const SUPPORT_COLUMN_FLAVORS = ['group', 'page'];
+
+interface AddColumnProps {
+    block: BlockImplInstance;
+    columns: ArrayOperation<MapOperation<string>>;
+    columnConfig: Column;
+}
+
+export const addColumn = ({ block, columns, columnConfig }: AddColumnProps) => {
+    const content = block.getContent();
+
+    const column_id = nanoid(16);
+    const config = serializeColumnConfig({
+        ...columnConfig,
+        id: column_id,
+    });
+
+    const db_column = content.createMap<string>();
+    // @ts-ignore TODO: don't know why
+    db_column.set('id', column_id);
+    // @ts-ignore TODO: don't know why
+    db_column.set('config', config);
+    columns?.insert(columns.length, [db_column]);
+};
+
+/**
+ * @deprecated
+ */
+export const getOrInitBlockContentColumnsField = (
+    block: BlockImplInstance
+): ArrayOperation<MapOperation<string>> | undefined => {
+    if (!SUPPORT_COLUMN_FLAVORS.includes(block.flavor)) {
+        return undefined;
+    }
+    const content = block.getContent();
+    if (!content.has('columns')) {
+        const columns = content.createArray();
+        content.set('columns', columns);
+        DEFAULT_COLUMNS.forEach(col => {
+            addColumn({
+                block,
+                columns: columns.asArray() as ArrayOperation<
+                    MapOperation<string>
+                >,
+                columnConfig: col,
+            });
+        });
+    }
+    return content.get('columns')?.asArray();
+};
+
+export const getBlockColumns = (
+    block: BlockImplInstance
+): Column[] | undefined => {
+    const columns = getOrInitBlockContentColumnsField(block)?.map<Column>(
+        column => {
+            const config_string = column.get('config') as unknown as string;
+            return {
+                id: column.get('id'),
+                ...(config_string
+                    ? deserializeColumnConfig(config_string)
+                    : {}),
+            } as Column;
+        }
+    );
+    return columns;
+};
+
+export type {
+    Column,
+    ContentColumn,
+    NumberColumn,
+    EnumColumn,
+    DateColumn,
+    BooleanColumn,
+    FileColumn,
+    ContentColumnValue,
+    NumberColumnValue,
+    EnumColumnValue,
+    BooleanColumnValue,
+    DateColumnValue,
+    FileColumnValue,
+    StringColumnValue,
+} from './types';
+export { ColumnType } from './types';
+export type { DefaultColumnsValue } from './default-config';
+
+export {
+    isContentColumn,
+    isDateColumn,
+    isFileColumn,
+    isNumberColumn,
+    isEnumColumn,
+    isStringColumn,
+    isBooleanColumn,
+} from './utils';

libs/datasource/db-service/src/services/editor-block/utils/column/types.ts

@@ -0,0 +1,204 @@
+/** Column */
+import type { CSSProperties } from 'react';
+
+export enum ColumnType {
+    /**
+     * the content of the text base block
+     */
+    content = 'content',
+    number = 'number',
+    enum = 'enum',
+    date = 'date',
+    boolean = 'boolean',
+    file = 'file',
+    string = 'string',
+}
+interface BaseColumn {
+    id?: string;
+    name: string;
+    /**
+     * key when assigning
+     */
+    key: string;
+    /**
+     * Properties used by the program, not as the display column of the Table
+     * @deprecated
+     */
+    innerColumn?: boolean;
+    /**
+     * Whether to support adding to the tag below the block
+     */
+    supportAsTag?: boolean;
+}
+export interface ContentColumn extends BaseColumn {
+    type: ColumnType.content;
+}
+
+export interface NumberColumn extends BaseColumn {
+    type: ColumnType.number;
+    /** not implemented */
+    format: 'number' | 'percent';
+}
+
+export interface SelectOption {
+    id: string;
+    name: string;
+    background?: string;
+    value: string | boolean;
+    color?: string;
+}
+
+export interface EnumColumn extends BaseColumn {
+    type: ColumnType.enum;
+    options: SelectOption[];
+    /**
+     * Limit the number of choices, if it is 1, it is a single choice
+     */
+    multiple: number;
+}
+
+export interface DateColumn extends BaseColumn {
+    type: ColumnType.date;
+    /**
+     * Date format, such as: YYYY-MM-DD hh:mm:ss
+     *ref: https://date-fns.org/v2.28.0/docs/format
+     */
+    format: string;
+}
+
+export interface BooleanColumn extends BaseColumn {
+    type: ColumnType.boolean;
+    options?: SelectOption[];
+    sorter?: boolean;
+}
+
+export interface FileColumn extends BaseColumn {
+    type: ColumnType.file;
+    /**
+     *ref: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file#accept
+     */
+    accept?: string;
+    /**
+     * Limit the number of choices, you can only choose one at a time, you can choose multiple times
+     */
+    multiple: number;
+}
+
+export interface StringColumn extends BaseColumn {
+    type: ColumnType.string;
+    mode: 'text' | 'url';
+}
+
+/**
+ * @deprecated
+ */
+export type Column =
+    | ContentColumn
+    | NumberColumn
+    | EnumColumn
+    | DateColumn
+    | BooleanColumn
+    | FileColumn
+    | StringColumn;
+
+/**
+ * ColumnValue
+ * @deprecated
+ */
+export interface ContentColumnValue {
+    value: Array<{ text: string; bold?: boolean }>;
+}
+
+/**
+ * @deprecated
+ */
+export interface NumberColumnValue {
+    value: number;
+}
+
+/**
+ * @deprecated
+ */
+export interface EnumColumnValue<T = string> {
+    value: T[];
+}
+
+/**
+ * @deprecated
+ */
+type Timestamp = number;
+
+/**
+ * @deprecated
+ */
+export interface DateColumnValue {
+    value: Timestamp;
+}
+
+/**
+ * @deprecated
+ */
+export interface BooleanColumnValue {
+    value: boolean;
+}
+
+/**
+ * @deprecated
+ */
+type FileBlockId = string;
+
+/**
+ * @deprecated
+ */
+type UrlString = string;
+
+/**
+ * @deprecated
+ */
+export interface FileColumnValue {
+    value: FileBlockId;
+    url?: UrlString;
+    name: string;
+    /**
+     * the size of the file in bytes
+     */
+    size: number;
+    /**
+     * ref file.type: https://developer.mozilla.org/en-US/docs/Web/API/File
+     */
+    type: string;
+    height?: number;
+    width?: number;
+}
+
+/**
+ * @deprecated
+ */
+export interface StringColumnValue {
+    value: string;
+    name?: string;
+}
+
+export interface CommentColumnValue {
+    pageId: string;
+    attachedToBlocksIds?: string[];
+    resolve: boolean;
+    resolveUserId?: string;
+    finishTime?: number;
+}
+export type FilterConstraint = Array<{
+    key: string;
+    checked: boolean;
+    type: string;
+    fieldValue: string;
+    opSelectValue: string;
+    valueSelectValue:
+        | string
+        | string[]
+        | { title?: string; value?: string | boolean }[];
+}>;
+
+export type SorterConstraint = Array<{
+    field: string;
+    rule: string;
+}>;

libs/datasource/db-service/src/services/editor-block/utils/column/utils.ts

@@ -0,0 +1,38 @@
+import type {
+    Column,
+    ContentColumn,
+    StringColumn,
+    NumberColumn,
+    EnumColumn,
+    DateColumn,
+    FileColumn,
+    BooleanColumn,
+} from './types';
+
+export const isContentColumn = (column: Column): column is ContentColumn => {
+    return column.type === 'content';
+};
+
+export const isStringColumn = (column: Column): column is StringColumn => {
+    return column.type === 'string';
+};
+
+export const isNumberColumn = (column: Column): column is NumberColumn => {
+    return column.type === 'number';
+};
+
+export const isEnumColumn = (column: Column): column is EnumColumn => {
+    return column.type === 'enum';
+};
+
+export const isDateColumn = (column: Column): column is DateColumn => {
+    return column.type === 'date';
+};
+
+export const isFileColumn = (column: Column): column is FileColumn => {
+    return column.type === 'file';
+};
+
+export const isBooleanColumn = (column: Column): column is BooleanColumn => {
+    return column.type === 'boolean';
+};

libs/datasource/db-service/src/services/editor-block/utils/common.ts

@@ -0,0 +1,58 @@
+import { BlockImplInstance } from '@toeverything/datasource/jwt';
+
+type Condition = (block: BlockImplInstance | undefined) => boolean;
+/**
+ * Find the block closest to the block up
+ * @param block
+ * @param condition conditional function, return true to indicate found
+ * @returns
+ */
+export const getClosestBlock = (
+    block: BlockImplInstance,
+    condition: Condition
+): BlockImplInstance | undefined => {
+    let group: BlockImplInstance | undefined = block;
+    while (!condition(group)) {
+        group = group?.parent;
+    }
+    return group;
+};
+
+/**
+ * Find the closest Page up
+ * @param block
+ * @returns
+ */
+export const getClosestPage = (
+    block: BlockImplInstance
+): BlockImplInstance | undefined => {
+    return getClosestBlock(block, block => {
+        return !block || block.flavor === 'page';
+    });
+};
+
+/**
+ * Find the closest group up
+ * @param block
+ * @returns
+ */
+export const getClosestGroup = (
+    block: BlockImplInstance
+): BlockImplInstance | undefined => {
+    return getClosestBlock(block, block => {
+        return !block || block.flavor === 'group';
+    });
+};
+
+/**
+ * Look up the group or page closest to the block
+ * @param block
+ * @returns
+ */
+export const getClosestGroupOrPage = (
+    block: BlockImplInstance
+): BlockImplInstance | undefined => {
+    return getClosestBlock(block, block => {
+        return !block || ['group', 'page'].includes(block?.flavor);
+    });
+};

libs/datasource/db-service/src/services/editor-block/utils/index.ts

@@ -0,0 +1,57 @@
+import type { BlockImplInstance } from '@toeverything/datasource/jwt';
+import { ReturnEditorBlock } from '../types';
+import { getClosestGroup } from './common';
+import { getBlockColumns } from './column';
+
+interface DbBlock2BusinessBlockProps {
+    workspace: string;
+    dbBlock?: BlockImplInstance | null;
+}
+
+export const dbBlock2BusinessBlock = ({
+    workspace,
+    dbBlock,
+}: DbBlock2BusinessBlockProps): ReturnEditorBlock | null => {
+    if (!dbBlock) {
+        return null;
+    }
+    const block = {} as ReturnEditorBlock;
+    block.id = dbBlock.id;
+    block.type = dbBlock.flavor;
+    block.workspace = workspace;
+    block.parentId = dbBlock.parent?.id;
+    block.closestGroupId = getClosestGroup(dbBlock)?.id;
+    block.children = dbBlock.children || [];
+    block.properties = dbBlock.getDecorations();
+    block.created = dbBlock.created;
+    block.lastUpdated = dbBlock.lastUpdated;
+    block.columns = getBlockColumns(dbBlock);
+    block.creator = dbBlock.creator;
+    return block;
+};
+
+export {
+    getOrInitBlockContentColumnsField,
+    serializeColumnConfig,
+    deserializeColumnConfig,
+    addColumn,
+    ColumnType,
+    isBooleanColumn,
+    isContentColumn,
+    isDateColumn,
+    isFileColumn,
+    isNumberColumn,
+    isEnumColumn,
+    isStringColumn,
+} from './column';
+export type {
+    Column,
+    DefaultColumnsValue,
+    ContentColumnValue,
+    NumberColumnValue,
+    EnumColumnValue,
+    BooleanColumnValue,
+    DateColumnValue,
+    FileColumnValue,
+    StringColumnValue,
+} from './column';

libs/datasource/db-service/src/services/file/index.ts

@@ -0,0 +1,52 @@
+import { ServiceBaseClass } from '../base';
+
+interface CreateParams {
+    file: File;
+    workspace: string;
+}
+
+interface ImageResult {
+    id: string;
+    url: string;
+}
+export class FileService extends ServiceBaseClass {
+    urlMap: Record<string, string> = {};
+
+    async create(params: CreateParams): Promise<ImageResult> {
+        const { file, workspace } = params;
+        // Get the current workspace block
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        // Get the workspace database link
+        const db = await this.database.getDatabase(workspace);
+        const binary = await file.arrayBuffer();
+        // create a block of type file
+        const file_db_block = await db.get('binary', {
+            flavor: 'file',
+            binary,
+        });
+        // add the file block to the workspace
+        workspace_db_block.append(file_db_block);
+        // cache the file
+        const file_key = file_db_block.id + workspace;
+        this.urlMap[file_key] = URL.createObjectURL(new Blob([binary]));
+        return { id: file_db_block.id, url: this.urlMap[file_key] };
+    }
+    async get(file_block_id: string, workspace: string): Promise<ImageResult> {
+        const file_key = file_block_id + workspace;
+        if (this.urlMap[file_key]) {
+            // lookup cache
+            return { id: file_block_id, url: this.urlMap[file_key] };
+        }
+        const db = await this.database.getDatabase(workspace);
+        const file_block = await db.get(file_block_id as 'binary');
+        const file_buffer = file_block.getBinary();
+        if (file_buffer) {
+            this.urlMap[file_key] = URL.createObjectURL(
+                new Blob([file_buffer])
+            );
+        } else {
+            this.urlMap[file_key] = '';
+        }
+        return { id: file_block.id, url: this.urlMap[file_key] };
+    }
+}

libs/datasource/db-service/src/services/index.ts

@@ -0,0 +1,108 @@
+import { DiContainer } from '@toeverything/utils';
+import type { RegisterDependencyConfig } from '@toeverything/utils';
+import { Database } from './database';
+import { PageTree } from './workspace/page-tree';
+import { UserConfig } from './workspace/user-config';
+import { EditorBlock } from './editor-block';
+import { FileService } from './file';
+import { CommentService } from './comment';
+
+export type {
+    CreateEditorBlock,
+    ReturnEditorBlock,
+    GetEditorBlock,
+    DeleteEditorBlock,
+    UpdateEditorBlock,
+    BlockFlavors,
+    BlockFlavorKeys,
+    Column,
+    ContentColumn,
+    NumberColumn,
+    EnumColumn,
+    DateColumn,
+    BooleanColumn,
+    FileColumn,
+    DefaultColumnsValue,
+    ContentColumnValue,
+    NumberColumnValue,
+    EnumColumnValue,
+    BooleanColumnValue,
+    DateColumnValue,
+    FileColumnValue,
+    StringColumnValue,
+} from './editor-block';
+export {
+    ColumnType,
+    isBooleanColumn,
+    isContentColumn,
+    isDateColumn,
+    isFileColumn,
+    isNumberColumn,
+    isEnumColumn,
+    isStringColumn,
+} from './editor-block';
+
+export interface DbServicesMap {
+    editorBlock: EditorBlock;
+    pageTree: PageTree;
+    userConfig: UserConfig;
+    file: FileService;
+    commentService: CommentService;
+}
+
+interface RegisterDependencyConfigWithName extends RegisterDependencyConfig {
+    callName: string;
+}
+
+const dbServiceConfig: RegisterDependencyConfigWithName[] = [
+    {
+        type: 'value',
+        callName: 'database',
+        token: Database,
+        value: new Database({}),
+        dependencies: [],
+    },
+    {
+        type: 'class',
+        callName: 'editorBlock',
+        token: EditorBlock,
+        value: EditorBlock,
+        dependencies: [{ token: Database }],
+    },
+    {
+        type: 'class',
+        callName: 'pageTree',
+        token: PageTree,
+        value: PageTree,
+        dependencies: [{ token: Database }],
+    },
+    {
+        type: 'class',
+        callName: 'userConfig',
+        token: UserConfig,
+        value: UserConfig,
+        dependencies: [{ token: Database }, { token: PageTree, lazy: true }],
+    },
+    {
+        type: 'class',
+        callName: 'file',
+        token: FileService,
+        value: FileService,
+        dependencies: [{ token: Database }],
+    },
+    {
+        type: 'class',
+        callName: 'commentService',
+        token: CommentService,
+        value: CommentService,
+        dependencies: [{ token: EditorBlock }],
+    },
+];
+
+export const serviceMapByCallName = dbServiceConfig.reduce((acc, cur) => {
+    acc[cur.callName] = cur;
+    return acc;
+}, {} as Record<string, RegisterDependencyConfigWithName>);
+
+export const diContainer = new DiContainer();
+diContainer.register(dbServiceConfig);

libs/datasource/db-service/src/services/workspace/page-tree.ts

@@ -0,0 +1,225 @@
+import type { BlockClientInstance } from '@toeverything/datasource/jwt';
+import { PAGE_TREE } from '../../utils';
+import type { ReturnUnobserve } from '../database/observer';
+import { ServiceBaseClass } from '../base';
+import { TreeItem } from './types';
+
+export type ObserveCallback = () => void;
+
+export class PageTree extends ServiceBaseClass {
+    private async fetch_page_tree<TreeItem>(workspace: string) {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        const page_tree_config =
+            workspace_db_block.getDecoration<TreeItem[]>(PAGE_TREE);
+        return page_tree_config;
+    }
+
+    async getPageTree<TreeItem>(workspace: string): Promise<TreeItem[]> {
+        try {
+            const page_tree = await this.fetch_page_tree(workspace);
+            if (page_tree && page_tree.length) {
+                const db = await this.database.getDatabase(workspace);
+
+                const pages = await update_tree_items_title(
+                    db,
+                    page_tree as [],
+                    {}
+                );
+                return pages;
+            }
+        } catch (e) {
+            console.error(e);
+        }
+
+        return [];
+    }
+
+    /** @deprecated should implement more fine-grained crud methods instead of replacing each time with a new array */
+    async setPageTree<TreeItem>(workspace: string, treeData: TreeItem[]) {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        workspace_db_block.setDecoration(PAGE_TREE, treeData);
+    }
+
+    async addPage<TreeItem>(workspace: string, treeData: TreeItem[] | string) {
+        // TODO: rewrite
+        if (typeof treeData === 'string') {
+            await this.setPageTree(workspace, [{ id: treeData, children: [] }]);
+        }
+    }
+    async removePage(workspace: string, blockId: string) {
+        const dbBlock = await this.getBlock(workspace, blockId);
+        await dbBlock?.remove();
+    }
+
+    async addPageToWorkspacee(
+        target_workspace_id: string,
+        new_page_id: string
+    ) {
+        const items = await this.getPageTree<TreeItem>(target_workspace_id);
+        await this.setPageTree(target_workspace_id, [
+            { id: new_page_id, children: [] },
+            ...items,
+        ]);
+    }
+
+    async addChildPageToWorkspace(
+        target_workspace_id: string,
+        parent_page_id: string,
+        new_page_id: string
+    ) {
+        const pages = await this.getPageTree<TreeItem>(target_workspace_id);
+        this.build_items_for_child_page(parent_page_id, new_page_id, pages);
+
+        await this.setPageTree<TreeItem>(target_workspace_id, [...pages]);
+    }
+    async addPrevPageToWorkspace(
+        target_workspace_id: string,
+        parent_page_id: string,
+        new_page_id: string
+    ) {
+        const pages = await this.getPageTree<TreeItem>(target_workspace_id);
+        this.build_items_for_prev_page(parent_page_id, new_page_id, pages);
+        await this.setPageTree<TreeItem>(target_workspace_id, [...pages]);
+    }
+    async addNextPageToWorkspace(
+        target_workspace_id: string,
+        parent_page_id: string,
+        new_page_id: string
+    ) {
+        const pages = await this.getPageTree<TreeItem>(target_workspace_id);
+        this.build_items_for_next_page(parent_page_id, new_page_id, pages);
+        await this.setPageTree<TreeItem>(target_workspace_id, [...pages]);
+    }
+    private build_items_for_next_page(
+        parent_page_id: string,
+        new_page_id: string,
+        children: TreeItem[]
+    ) {
+        for (let i = 0; i < children.length; i++) {
+            const child_page = children[i];
+            if (child_page.id === parent_page_id) {
+                const new_page = {
+                    id: new_page_id,
+                    title: 'Untitled',
+                    children: [] as TreeItem[],
+                };
+                children = children.splice(i + 1, 0, new_page);
+            } else if (child_page.children && child_page.children.length) {
+                this.build_items_for_next_page(
+                    parent_page_id,
+                    new_page_id,
+                    child_page.children
+                );
+            }
+        }
+    }
+    private build_items_for_prev_page(
+        parent_page_id: string,
+        new_page_id: string,
+        children: TreeItem[]
+    ) {
+        for (let i = 0; i < children.length; i++) {
+            const child_page = children[i];
+            if (child_page.id === parent_page_id) {
+                const new_page = {
+                    id: new_page_id,
+                    title: 'Untitled',
+                    children: [] as TreeItem[],
+                };
+                children = children.splice(i - 1, 0, new_page);
+            } else if (child_page.children && child_page.children.length) {
+                this.build_items_for_prev_page(
+                    parent_page_id,
+                    new_page_id,
+                    child_page.children
+                );
+            }
+        }
+    }
+    private build_items_for_child_page(
+        parent_page_id: string,
+        new_page_id: string,
+        children: TreeItem[]
+    ) {
+        for (let i = 0; i < children.length; i++) {
+            const child_page = children[i];
+            if (child_page.id === parent_page_id) {
+                child_page.children = child_page.children || [];
+                child_page.children.push({
+                    id: new_page_id,
+                    title: 'Untitled',
+                    children: [],
+                });
+            } else if (child_page.children && child_page.children.length) {
+                this.build_items_for_child_page(
+                    parent_page_id,
+                    new_page_id,
+                    child_page.children
+                );
+            }
+        }
+    }
+    // TODO: handles unobserve
+    async observe(
+        { workspace, page }: { workspace: string; page: string },
+        callback: ObserveCallback
+    ): Promise<ReturnUnobserve> {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        const unobserveWorkspace = await this._observe(
+            workspace,
+            workspace_db_block.id,
+            (states, block) => {
+                callback();
+            }
+        );
+        const unobservePage = await this._observe(
+            workspace,
+            page,
+            (states, block) => {
+                callback();
+            }
+        );
+
+        return () => {
+            unobserveWorkspace();
+            unobservePage();
+        };
+    }
+
+    async unobserve({ workspace }: { workspace: string }) {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        await this._unobserve(workspace, workspace_db_block.id);
+    }
+}
+
+async function update_tree_items_title<
+    TreeItem extends { id: string; title: string; children: TreeItem[] }
+>(
+    db: BlockClientInstance,
+    items: TreeItem[],
+    cache: Record<string, string>
+): Promise<TreeItem[]> {
+    for (const item of items) {
+        if (cache[item.id]) {
+            item.title = cache[item.id];
+        } else {
+            const page = await db.get(item.id as 'page');
+            item.title =
+                page
+                    .getDecoration<{ value: Array<{ text: string }> }>('text')
+                    ?.value?.map(v => v.text)
+                    .join('') || 'Untitled';
+            cache[item.id] = item.title;
+        }
+
+        if (item.children.length) {
+            item.children = await update_tree_items_title(
+                db,
+                item.children,
+                cache
+            );
+        }
+    }
+
+    return [...items];
+}

libs/datasource/db-service/src/services/workspace/types.ts

@@ -0,0 +1,13 @@
+interface TreeItem {
+    id: string;
+    title: string;
+    children?: TreeItem[];
+}
+
+interface PageConfigItem {
+    id: string;
+    title?: string;
+    lastOpenTime: number;
+}
+
+export type { TreeItem, PageConfigItem };

libs/datasource/db-service/src/services/workspace/user-config.ts

@@ -0,0 +1,124 @@
+import { RECENT_PAGES, WORKSPACE_CONFIG } from '../../utils';
+import { ServiceBaseClass } from '../base';
+import { ObserveCallback, ReturnUnobserve } from '../database';
+import { PageTree } from './page-tree';
+import { PageConfigItem } from './types';
+
+/** Operate the user configuration at the workspace level */
+export class UserConfig extends ServiceBaseClass {
+    private async fetch_recent_pages(
+        workspace: string
+    ): Promise<Record<string, Array<PageConfigItem>>> {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        const recent_work_pages =
+            workspace_db_block.getDecoration<
+                Record<string, Array<PageConfigItem>>
+            >(RECENT_PAGES) || {};
+        return recent_work_pages;
+    }
+
+    private async save_recent_pages(
+        workspace: string,
+        recentPages: Record<string, Array<PageConfigItem>>
+    ) {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        workspace_db_block.setDecoration(RECENT_PAGES, recentPages);
+    }
+
+    async getUserInitialPage(
+        workspace: string,
+        userId: string
+    ): Promise<string> {
+        const recent_pages = await this.getRecentPages(workspace, userId);
+        if (recent_pages.length > 0) {
+            return recent_pages[0].id;
+        }
+
+        const db = await this.database.getDatabase(workspace);
+        const new_page = await db.get('page');
+
+        await this.get_dependency(PageTree).addPage(workspace, new_page.id);
+        await this.addRecentPage(workspace, userId, new_page.id);
+        return new_page.id;
+    }
+
+    async getRecentPages(
+        workspace: string,
+        userId: string,
+        topNumber = 5
+    ): Promise<PageConfigItem[]> {
+        const recent_work_pages = await this.fetch_recent_pages(workspace);
+        const recent_pages = (recent_work_pages[userId] || []).slice(
+            0,
+            topNumber
+        );
+        const db = await this.database.getDatabase(workspace);
+        for (const item of recent_pages) {
+            const page = await db.get(item.id as 'page');
+            item.title =
+                page
+                    .getDecoration<{ value: Array<{ text: string }> }>('text')
+                    ?.value?.map(v => v.text)
+                    .join('') || 'Untitled';
+        }
+        return recent_pages;
+    }
+
+    async addRecentPage(workspace: string, userId: string, pageId: string) {
+        const recent_work_pages = await this.fetch_recent_pages(workspace);
+        let recent_pages = recent_work_pages[userId] || [];
+        recent_pages = recent_pages.filter(item => item.id !== pageId);
+        recent_pages.unshift({
+            id: pageId,
+            lastOpenTime: Date.now(),
+        });
+        recent_work_pages[userId] = recent_pages;
+        await this.save_recent_pages(workspace, recent_work_pages);
+    }
+
+    async removePage(workspace: string, pageId: string) {
+        const recent_work_pages = await this.fetch_recent_pages(workspace);
+        for (const key in recent_work_pages) {
+            recent_work_pages[key] = recent_work_pages[key].filter(
+                item => item.id !== pageId
+            );
+        }
+        await this.save_recent_pages(workspace, recent_work_pages);
+    }
+
+    async observe(
+        { workspace }: { workspace: string },
+        callback: ObserveCallback
+    ): Promise<ReturnUnobserve> {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        const unobserveWorkspace = await this._observe(
+            workspace,
+            workspace_db_block.id,
+            (states, block) => {
+                callback(states, block);
+            }
+        );
+
+        return () => {
+            unobserveWorkspace();
+        };
+    }
+
+    async unobserve({ workspace }: { workspace: string }) {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        await this._unobserve(workspace, workspace_db_block.id);
+    }
+
+    async getWorkspaceName(workspace: string): Promise<string> {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        const workspaceName =
+            workspace_db_block.getDecoration<string>(WORKSPACE_CONFIG) ||
+            workspace_db_block.id;
+        return workspaceName;
+    }
+
+    async setWorkspaceName(workspace: string, workspaceName: string) {
+        const workspace_db_block = await this.getWorkspaceDbBlock(workspace);
+        workspace_db_block.setDecoration(WORKSPACE_CONFIG, workspaceName);
+    }
+}

libs/datasource/db-service/src/utils/constants.ts

@@ -0,0 +1,5 @@
+// blockdb configuration item related property name key
+export const PAGE_TREE = 'page_tree';
+export const RECENT_PAGES = 'activities';
+export const WORKSPACE_COMMENTS = 'workspace_comments';
+export const WORKSPACE_CONFIG = 'workspace_config';

libs/datasource/db-service/src/utils/index.ts

@@ -0,0 +1 @@
+export * from './constants';

libs/datasource/db-service/tsconfig.json

@@ -0,0 +1,24 @@
+{
+    "extends": "../../../tsconfig.base.json",
+    "compilerOptions": {
+        "allowSyntheticDefaultImports": true,
+        "forceConsistentCasingInFileNames": true,
+        "strict": true,
+        "noImplicitOverride": true,
+        "noPropertyAccessFromIndexSignature": true,
+        "noImplicitReturns": true,
+        "noFallthroughCasesInSwitch": true,
+        // "noImplicitAny": true,
+        "strictNullChecks": true
+    },
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        },
+        {
+            "path": "./tsconfig.spec.json"
+        }
+    ]
+}

libs/datasource/db-service/tsconfig.lib.json

@@ -0,0 +1,22 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "files": [
+        "../../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+        "../../../node_modules/@nrwl/react/typings/image.d.ts"
+    ],
+    "exclude": [
+        "**/*.spec.ts",
+        "**/*.test.ts",
+        "**/*.spec.tsx",
+        "**/*.test.tsx",
+        "**/*.spec.js",
+        "**/*.test.js",
+        "**/*.spec.jsx",
+        "**/*.test.jsx"
+    ],
+    "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]
+}

libs/datasource/db-service/tsconfig.spec.json

@@ -0,0 +1,19 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "module": "commonjs",
+        "types": ["jest", "node"]
+    },
+    "include": [
+        "**/*.test.ts",
+        "**/*.spec.ts",
+        "**/*.test.tsx",
+        "**/*.spec.tsx",
+        "**/*.test.js",
+        "**/*.spec.js",
+        "**/*.test.jsx",
+        "**/*.spec.jsx",
+        "**/*.d.ts"
+    ]
+}

libs/datasource/feature-flags/.babelrc

@@ -0,0 +1,12 @@
+{
+    "presets": [
+        [
+            "@nrwl/react/babel",
+            {
+                "runtime": "automatic",
+                "useBuiltIns": "usage"
+            }
+        ]
+    ],
+    "plugins": []
+}

libs/datasource/feature-flags/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "extends": ["plugin:@nrwl/nx/react", "../../../.eslintrc.json"],
+    "ignorePatterns": ["!**/*"],
+    "overrides": [
+        {
+            "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.ts", "*.tsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.js", "*.jsx"],
+            "rules": {}
+        }
+    ]
+}

libs/datasource/feature-flags/README.md

@@ -0,0 +1,55 @@
+# Feature Flags
+
+## Usage
+
+-   set provider
+
+```tsx
+import { FeatureFlagsProvider } from '@toeverything/datasource/feature-flags';
+
+const App = () => {
+    return (
+        <FeatureFlagsProvider>
+            <Page />
+        </FeatureFlagsProvider>
+    );
+};
+```
+
+-   use flag
+
+```ts
+import { useFlag } from '@toeverything/datasource/feature-flags';
+
+const App = () => {
+    //                   flag key, default value
+    const flag1 = useFlag('flag1', false);
+    // ^? boolean
+    const flag2 = useFlag<boolean>('flag2');
+    // ^? boolean | undefined
+    return flag1 && <div>The flag1 is enable!</div>;
+};
+```
+
+## How to add a new feature flag?
+
+-   Enter [Portal](https://portal.featureflag.co/switch-manage)
+-   Click `New Switch` button
+    -   Input `Switch Name`, and Enter
+    -   Adjust data and save
+
+# Test flags
+
+**When entering development mode feature flag will NOT be updated in real time**
+
+-   `activateFfcDevMode()` play with feature flags locally
+-   `quitFfcDevMode()` quit dev mode
+
+## Running unit tests
+
+Run `nx test datasource/feature-flags` to execute the unit tests via [Jest](https://jestjs.io).
+
+## References
+
+-   [Feature Flag Portal](https://portal.featureflag.co/)
+-   [Feature Flag React Web APP DEMO](https://featureflag.moyincloud.com/ksrm/React_Web_APP.html)

libs/datasource/feature-flags/jest.config.ts

@@ -0,0 +1,10 @@
+/* eslint-disable */
+export default {
+    displayName: 'datasource-feature-flags',
+    preset: '../../../jest.preset.js',
+    transform: {
+        '^.+\\.[tj]sx?$': 'babel-jest',
+    },
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+    coverageDirectory: '../../../coverage/libs/datasource/feature-flags',
+};

libs/datasource/feature-flags/package.json

@@ -0,0 +1,8 @@
+{
+    "name": "@toeverything/datasource/feature-flags",
+    "version": "0.0.1",
+    "license": "MIT",
+    "dependencies": {
+        "ffc-js-client-side-sdk": "^1.1.4"
+    }
+}

libs/datasource/feature-flags/project.json

@@ -0,0 +1,45 @@
+{
+    "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "libs/datasource/feature-flags/src",
+    "projectType": "library",
+    "tags": ["library:feature-flags"],
+    "targets": {
+        "build": {
+            "executor": "@nrwl/web:rollup",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/libs/datasource/feature-flags",
+                "tsConfig": "libs/datasource/feature-flags/tsconfig.lib.json",
+                "project": "libs/datasource/feature-flags/package.json",
+                "entryFile": "libs/datasource/feature-flags/src/index.ts",
+                "external": ["react/jsx-runtime"],
+                "rollupConfig": "@nrwl/react/plugins/bundle-rollup",
+                "compiler": "babel",
+                "assets": [
+                    {
+                        "glob": "libs/datasource/feature-flags/README.md",
+                        "input": ".",
+                        "output": "."
+                    }
+                ]
+            }
+        },
+        "lint": {
+            "executor": "@nrwl/linter:eslint",
+            "outputs": ["{options.outputFile}"],
+            "options": {
+                "lintFilePatterns": [
+                    "libs/datasource/feature-flags/**/*.{ts,tsx,js,jsx}"
+                ]
+            }
+        },
+        "test": {
+            "executor": "@nrwl/jest:jest",
+            "outputs": ["coverage/libs/datasource/feature-flags"],
+            "options": {
+                "jestConfig": "libs/datasource/feature-flags/jest.config.ts",
+                "passWithNoTests": true
+            }
+        }
+    }
+}

libs/datasource/feature-flags/src/Context.tsx

@@ -0,0 +1,63 @@
+import ffcClient, { Ffc } from 'ffc-js-client-side-sdk';
+import type {
+    IFeatureFlagSet,
+    IOption,
+} from 'ffc-js-client-side-sdk/esm/types';
+import { createContext, ReactNode, useEffect, useState } from 'react';
+import { config } from './config';
+
+/**
+ * Init `ffcClient`
+ * Ported from https://github.com/feature-flags-co/ffc-js-client-side-sdk-react-jotai-demo/blob/main/src/ffc/hooks.ts
+ * @private
+ */
+export const useInitFfcEffect = (featureFlagClient: Ffc, option: IOption) => {
+    const [flags, setFlags] = useState<Record<string, IFeatureFlagSet>>({});
+
+    useEffect(() => {
+        featureFlagClient.init(option);
+
+        featureFlagClient.on('ff_update', (changes: unknown[]) => {
+            if (changes.length) {
+                setFlags(featureFlagClient.getAllFeatureFlags());
+            }
+        });
+
+        featureFlagClient.waitUntilReady().then((data: unknown[]) => {
+            if (data.length) {
+                setFlags(featureFlagClient.getAllFeatureFlags());
+            }
+        });
+        return () => {
+            // FIX destroy ffcClient
+            // ffcClient.logout();
+        };
+    }, [featureFlagClient, option, setFlags]);
+    return flags;
+};
+
+export const FeatureFlagsContext = createContext<{
+    ffcClient: Ffc;
+    flags: IFeatureFlagSet;
+    // TODO use genErrorObj
+}>({} as any);
+
+/**
+ * @example
+ * ```ts
+ * const App = () => (
+ *   <FeatureFlagsProvider>
+ *       <Page />
+ *   </FeatureFlagsProvider>
+ * );
+ * ```
+ */
+export const FeatureFlagsProvider = ({ children }: { children: ReactNode }) => {
+    const flags = useInitFfcEffect(ffcClient, config);
+
+    return (
+        <FeatureFlagsContext.Provider value={{ ffcClient, flags }}>
+            {children}
+        </FeatureFlagsContext.Provider>
+    );
+};

libs/datasource/feature-flags/src/config.ts

@@ -0,0 +1,11 @@
+import type { IOption } from 'ffc-js-client-side-sdk/esm/types';
+
+export const config: IOption = {
+    secret: process.env['AFFINE_FEATURE_FLAG_TOKEN'],
+    anonymous: true,
+    // user: {
+    //   userName: 'the user's user name',
+    //   id: 'the user's unique identifier'
+    // }
+    devModePassword: '-',
+};

libs/datasource/feature-flags/src/index.ts

@@ -0,0 +1,111 @@
+// Feature Flags
+// Website https://featureflag.co/
+// SDK doc https://github.com/feature-flags-co/ffc-js-client-side-sdk
+// Demo https://github.com/feature-flags-co/ffc-js-client-side-sdk-react-jotai-demo
+
+import type { IUser } from 'ffc-js-client-side-sdk/esm/types';
+import { useContext } from 'react';
+import { FeatureFlagsContext } from './Context';
+
+/**
+ * The `ffcClient` should not export to external modules.
+ * Do not export this please.
+ *
+ * @private
+ */
+const useFfcClient = () => {
+    const { ffcClient } = useContext(FeatureFlagsContext);
+    return ffcClient;
+};
+
+/**
+ * NOTICE: The hook can no trigger the `Ffc.sendFeatureFlagInsight`,
+ * may lead to inaccurate telemetry data.
+ *
+ * Use it discreetly.
+ */
+const useFlags = () => {
+    const { flags } = useContext(FeatureFlagsContext);
+    return flags;
+};
+
+type UseFlagFn = (<T = unknown>(flag: string) => T | undefined) &
+    (<T = unknown>(flag: string, defaultValue: T) => T) &
+    // Workaround for infers boolean as true or false
+    // Remove this after the issue fixed
+    // See https://github.com/microsoft/TypeScript/issues/29400
+    ((flag: string, defaultValue: false) => boolean) &
+    ((flag: string, defaultValue: true) => boolean);
+
+/**
+ * Returns the specified flag.
+ *
+ * The parameter defaultValue should have the same data type with flag type.
+ *
+ * @public
+ * @example
+ * ```ts
+ * const App = () => {
+ *   const flag1 = useFlag('flag1', 'default value');
+ *   // ^? string
+ *   const flag2 = useFlag<boolean>('flag2');
+ *   // ^? boolean | undefined
+ *   return <div>flag1: {flag1}</div>
+ * }
+ * ```
+ */
+export const useFlag = ((flag: string, defaultValue: unknown) => {
+    const flags = useFlags();
+    const ffcClient = useFfcClient();
+    // @ts-expect-error This is a BUG for ffc-js-client. ffc-js-client supported typed flag now.
+    ffcClient.sendFeatureFlagInsight(flag, defaultValue);
+    const value: unknown = flags[flag];
+
+    if (defaultValue === undefined) {
+        // Can not guess the type, return the flag value directly
+        return value;
+    }
+
+    if (value === undefined) {
+        return defaultValue;
+    }
+
+    if (typeof value !== typeof defaultValue) {
+        console.error(
+            `[Feature Flags] Flag "${flag}" type mismatch! Make ensure you have set the correct type in feature flag portal! flag type: "${typeof value}", defaultValue type: "${typeof defaultValue}"`,
+            'flag:',
+            value,
+            'defaultValue:',
+            defaultValue
+        );
+        return defaultValue;
+    }
+    return value;
+}) as UseFlagFn;
+
+/**
+ * Set the user after initialization
+ *
+ * If the user parameter cannot be passed by the init method,
+ * the following method can be used to set the user after initialization.
+ *
+ * See https://github.com/feature-flags-co/ffc-js-client-side-sdk#set-the-user-after-initialization
+ */
+export const useIdentifyUser = () => {
+    const ffcClient = useFfcClient();
+    return (user: IUser) => ffcClient.identify(user);
+};
+
+/**
+ * Set the user to anonymous user
+ *
+ * We can manually call the method logout, which will switch the current user back to anonymous user if exists already or create a new anonymous user.
+ *
+ * See https://github.com/feature-flags-co/ffc-js-client-side-sdk#set-the-user-to-anonymous-user
+ */
+export const useLogout = () => {
+    const ffcClient = useFfcClient();
+    return () => ffcClient.logout();
+};
+
+export { FeatureFlagsProvider } from './Context';

libs/datasource/feature-flags/tsconfig.json

@@ -0,0 +1,27 @@
+{
+    "extends": "../../../tsconfig.base.json",
+    "compilerOptions": {
+        "jsx": "react-jsx",
+        "allowJs": true,
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "forceConsistentCasingInFileNames": true,
+        "strict": true,
+        "strictNullChecks": true,
+        "noImplicitOverride": true,
+        "noPropertyAccessFromIndexSignature": true,
+        "noImplicitReturns": true,
+        "noFallthroughCasesInSwitch": true,
+        "importsNotUsedAsValues": "error"
+    },
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        },
+        {
+            "path": "./tsconfig.spec.json"
+        }
+    ]
+}

libs/datasource/feature-flags/tsconfig.lib.json

@@ -0,0 +1,23 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "files": [
+        "../../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+        "../../../node_modules/@nrwl/react/typings/image.d.ts"
+    ],
+    "exclude": [
+        "jest.config.ts",
+        "**/*.spec.ts",
+        "**/*.test.ts",
+        "**/*.spec.tsx",
+        "**/*.test.tsx",
+        "**/*.spec.js",
+        "**/*.test.js",
+        "**/*.spec.jsx",
+        "**/*.test.jsx"
+    ],
+    "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]
+}

libs/datasource/feature-flags/tsconfig.spec.json

@@ -0,0 +1,20 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "module": "commonjs",
+        "types": ["jest", "node"]
+    },
+    "include": [
+        "jest.config.ts",
+        "**/*.test.ts",
+        "**/*.spec.ts",
+        "**/*.test.tsx",
+        "**/*.spec.tsx",
+        "**/*.test.js",
+        "**/*.spec.js",
+        "**/*.test.jsx",
+        "**/*.spec.jsx",
+        "**/*.d.ts"
+    ]
+}

libs/datasource/jwt-rpc/.babelrc

@@ -0,0 +1,12 @@
+{
+    "presets": [
+        [
+            "@nrwl/react/babel",
+            {
+                "runtime": "automatic",
+                "useBuiltIns": "usage"
+            }
+        ]
+    ],
+    "plugins": []
+}

libs/datasource/jwt-rpc/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+    "extends": ["plugin:@nrwl/nx/react", "../../../.eslintrc.json"],
+    "ignorePatterns": ["!**/*"],
+    "overrides": [
+        {
+            "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.ts", "*.tsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.js", "*.jsx"],
+            "rules": {}
+        }
+    ]
+}

libs/datasource/jwt-rpc/jest.config.ts

@@ -0,0 +1,10 @@
+/* eslint-disable */
+export default {
+    displayName: 'datasource-jwt-rpc',
+    preset: '../../../jest.preset.js',
+    transform: {
+        '^.+\\.[tj]sx?$': 'babel-jest',
+    },
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+    coverageDirectory: '../../../coverage/libs/datasource/jwt-rpc',
+};

libs/datasource/jwt-rpc/package.json

@@ -0,0 +1,11 @@
+{
+    "name": "@toeverything/datasource/jwt-rpc",
+    "version": "0.0.1",
+    "license": "MIT",
+    "author": "DarkSky <darksky2048@gmail.com>",
+    "dependencies": {
+        "lib0": "^0.2.51",
+        "yjs": "^13.5.39",
+        "y-protocols": "^1.0.5"
+    }
+}

libs/datasource/jwt-rpc/project.json

@@ -0,0 +1,45 @@
+{
+    "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "libs/datasource/jwt-rpc/src",
+    "projectType": "library",
+    "tags": ["datasource:jwt-rpc"],
+    "targets": {
+        "build": {
+            "executor": "@nrwl/web:rollup",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/libs/datasource/jwt-rpc",
+                "tsConfig": "libs/datasource/jwt-rpc/tsconfig.lib.json",
+                "project": "libs/datasource/jwt-rpc/package.json",
+                "entryFile": "libs/datasource/jwt-rpc/src/index.ts",
+                "external": ["react/jsx-runtime"],
+                "rollupConfig": "@nrwl/react/plugins/bundle-rollup",
+                "compiler": "babel",
+                "assets": [
+                    {
+                        "glob": "libs/datasource/jwt-rpc/README.md",
+                        "input": ".",
+                        "output": "."
+                    }
+                ]
+            }
+        },
+        "lint": {
+            "executor": "@nrwl/linter:eslint",
+            "outputs": ["{options.outputFile}"],
+            "options": {
+                "lintFilePatterns": [
+                    "libs/datasource/jwt-rpc/**/*.{ts,tsx,js,jsx}"
+                ]
+            }
+        },
+        "test": {
+            "executor": "@nrwl/jest:jest",
+            "outputs": ["coverage/libs/datasource/jwt-rpc"],
+            "options": {
+                "jestConfig": "libs/datasource/jwt-rpc/jest.config.ts",
+                "passWithNoTests": true
+            }
+        }
+    }
+}

libs/datasource/jwt-rpc/src/connector.ts

@@ -0,0 +1,347 @@
+import * as Y from 'yjs';
+import * as bc from 'lib0/broadcastchannel';
+import * as time from 'lib0/time';
+import * as encoding from 'lib0/encoding';
+import * as decoding from 'lib0/decoding';
+import * as syncProtocol from 'y-protocols/sync';
+import * as awarenessProtocol from 'y-protocols/awareness';
+import * as math from 'lib0/math';
+
+import { WebsocketProvider } from './provider';
+import { Message } from './handler';
+
+export const readMessage = (
+    provider: WebsocketProvider,
+    buf: Uint8Array,
+    emitSynced: boolean
+): encoding.Encoder => {
+    const decoder = decoding.createDecoder(buf);
+    const encoder = encoding.createEncoder();
+    const messageType = decoding.readVarUint(decoder) as Message;
+    const messageHandler = provider.messageHandlers[messageType];
+    if (/** @type {any} */ messageHandler) {
+        messageHandler(encoder, decoder, provider, emitSynced, messageType);
+    } else {
+        console.error('Unable to compute message');
+    }
+    return encoder;
+};
+
+export const registerBroadcastSubscriber = (
+    provider: WebsocketProvider,
+    awareness: awarenessProtocol.Awareness,
+    document: Y.Doc
+) => {
+    const channel = provider.broadcastChannel;
+
+    const subscriber = (data: ArrayBuffer, origin: any) => {
+        if (origin !== provider) {
+            const encoder = readMessage(provider, new Uint8Array(data), false);
+            if (encoding.length(encoder) > 1) {
+                bc.publish(channel, encoding.toUint8Array(encoder), provider);
+            }
+        }
+    };
+
+    bc.subscribe(channel, subscriber);
+    let connected = true;
+
+    // send sync step1 to bc
+    // write sync step 1
+    const encoderSync = encoding.createEncoder();
+    encoding.writeVarUint(encoderSync, Message.sync);
+    syncProtocol.writeSyncStep1(encoderSync, document);
+    bc.publish(channel, encoding.toUint8Array(encoderSync), this);
+    // broadcast local state
+    const encoderState = encoding.createEncoder();
+    encoding.writeVarUint(encoderState, Message.sync);
+    syncProtocol.writeSyncStep2(encoderState, document);
+    bc.publish(channel, encoding.toUint8Array(encoderState), this);
+    // write queryAwareness
+    const encoderAwarenessQuery = encoding.createEncoder();
+    encoding.writeVarUint(encoderAwarenessQuery, Message.queryAwareness);
+    bc.publish(channel, encoding.toUint8Array(encoderAwarenessQuery), this);
+    // broadcast local awareness state
+    const encoderAwarenessState = encoding.createEncoder();
+    encoding.writeVarUint(encoderAwarenessState, Message.awareness);
+    encoding.writeVarUint8Array(
+        encoderAwarenessState,
+        awarenessProtocol.encodeAwarenessUpdate(awareness, [document.clientID])
+    );
+    bc.publish(channel, encoding.toUint8Array(encoderAwarenessState), this);
+
+    const broadcastMessage = (buf: ArrayBuffer) => {
+        if (connected) bc.publish(channel, buf, provider);
+    };
+
+    const disconnect = () => {
+        const encoder = encoding.createEncoder();
+        encoding.writeVarUint(encoder, Message.awareness);
+        encoding.writeVarUint8Array(
+            encoder,
+            awarenessProtocol.encodeAwarenessUpdate(
+                awareness,
+                [document.clientID],
+                new Map()
+            )
+        );
+        broadcastMessage(encoding.toUint8Array(encoder));
+        if (connected) {
+            bc.unsubscribe(channel, subscriber);
+            connected = false;
+        }
+    };
+
+    return { broadcastMessage, disconnect };
+};
+
+export const registerUpdateHandler = (
+    provider: WebsocketProvider,
+    awareness: awarenessProtocol.Awareness,
+    doc: Y.Doc,
+    broadcastMessage: (buf: ArrayBuffer) => void
+) => {
+    const beforeUnloadHandler = () => {
+        awarenessProtocol.removeAwarenessStates(
+            awareness,
+            [doc.clientID],
+            'window unload'
+        );
+    };
+
+    const awarenessUpdateHandler = ({ added, updated, removed }: any) => {
+        const changedClients = added.concat(updated).concat(removed);
+        const encoder = encoding.createEncoder();
+        encoding.writeVarUint(encoder, Message.awareness);
+        encoding.writeVarUint8Array(
+            encoder,
+            awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients)
+        );
+        broadcastMessage(encoding.toUint8Array(encoder));
+    };
+
+    //  Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel)
+    const documentUpdateHandler = (update: Uint8Array, origin: any) => {
+        if (origin !== provider) {
+            const encoder = encoding.createEncoder();
+            encoding.writeVarUint(encoder, Message.sync);
+            syncProtocol.writeUpdate(encoder, update);
+            broadcastMessage(encoding.toUint8Array(encoder));
+        }
+    };
+
+    if (typeof window !== 'undefined') {
+        window.addEventListener('beforeunload', beforeUnloadHandler);
+    } else if (typeof process !== 'undefined') {
+        process.on('exit', beforeUnloadHandler);
+    }
+
+    awareness.on('update', awarenessUpdateHandler);
+    doc.on('update', documentUpdateHandler);
+    return () => {
+        if (typeof window !== 'undefined') {
+            window.removeEventListener('beforeunload', beforeUnloadHandler);
+        } else if (typeof process !== 'undefined') {
+            process.off('exit', beforeUnloadHandler);
+        }
+
+        awareness.off('update', awarenessUpdateHandler);
+        doc.off('update', documentUpdateHandler);
+    };
+};
+
+enum WebSocketState {
+    disconnected = 0,
+    connecting,
+    connected,
+}
+
+// @todo - this should depend on awareness.outdatedTime
+const WEBSOCKET_RECONNECT = 30000;
+
+const _getToken = async (
+    remote: string,
+    token: string,
+    existsProtocol?: string,
+    reconnect = 3
+) => {
+    if (existsProtocol && reconnect > 0) {
+        return { protocol: existsProtocol };
+    }
+    const url = new URL(remote);
+    url.protocol = window.location.protocol;
+    return fetch(url, { method: 'POST', headers: { token } }).then(r =>
+        r.json()
+    );
+};
+
+export const registerWebsocket = (
+    provider: WebsocketProvider,
+    token: string,
+    resync = -1,
+    reconnect = 3,
+    existsProtocol?: string
+) => {
+    let state = WebSocketState.disconnected;
+    let lastMessageReceived = 0;
+
+    let websocket: WebSocket | undefined = undefined;
+
+    _getToken(provider.url, token, existsProtocol, reconnect)
+        .then(({ protocol }) => {
+            websocket = new WebSocket(provider.url, protocol);
+            websocket.binaryType = 'arraybuffer';
+            state = WebSocketState.connecting;
+
+            provider.synced = false;
+
+            websocket.onmessage = event => {
+                lastMessageReceived = time.getUnixTime();
+                const encoder = readMessage(
+                    provider,
+                    new Uint8Array(event.data),
+                    true
+                );
+                if (encoding.length(encoder) > 1) {
+                    websocket?.send(encoding.toUint8Array(encoder));
+                }
+            };
+            websocket.onerror = event => {
+                provider.emit('connection-error', [event, provider]);
+            };
+            websocket.onclose = event => {
+                provider.emit('connection-close', [event, provider]);
+                websocket = undefined;
+
+                if (
+                    [
+                        WebSocketState.connecting,
+                        WebSocketState.connected,
+                    ].includes(state)
+                ) {
+                    state = WebSocketState.disconnected;
+                    provider.synced = false;
+                    // update awareness (all users except local left)
+                    awarenessProtocol.removeAwarenessStates(
+                        provider.awareness,
+                        Array.from(
+                            provider.awareness.getStates().keys()
+                        ).filter(client => client !== provider.doc.clientID),
+                        provider
+                    );
+                    provider.emit('status', [{ status: 'disconnected' }]);
+                } else {
+                    provider.wsUnsuccessfulReconnects++;
+                }
+                if (reconnect <= 0) {
+                    provider.emit('lost-connection', []);
+                }
+                // Start with no reconnect timeout and increase timeout by
+                // using exponential backoff starting with 100ms
+                setTimeout(
+                    registerWebsocket,
+                    math.min(
+                        math.pow(2, provider.wsUnsuccessfulReconnects) * 100,
+                        provider.maxBackOffTime
+                    ),
+                    provider,
+                    token,
+                    resyncInterval,
+                    reconnect > 0 ? reconnect - 1 : 3,
+                    protocol
+                );
+            };
+            websocket.onopen = () => {
+                lastMessageReceived = time.getUnixTime();
+                state = WebSocketState.connected;
+                provider.wsUnsuccessfulReconnects = 0;
+                provider.emit('status', [{ status: 'connected' }]);
+                // always send sync step 1 when connected
+                const encoder = encoding.createEncoder();
+                encoding.writeVarUint(encoder, Message.sync);
+                syncProtocol.writeSyncStep1(encoder, provider.doc);
+                websocket?.send(encoding.toUint8Array(encoder));
+                // broadcast local awareness state
+                if (provider.awareness.getLocalState() !== null) {
+                    const encoderAwarenessState = encoding.createEncoder();
+                    encoding.writeVarUint(
+                        encoderAwarenessState,
+                        Message.awareness
+                    );
+                    encoding.writeVarUint8Array(
+                        encoderAwarenessState,
+                        awarenessProtocol.encodeAwarenessUpdate(
+                            provider.awareness,
+                            [provider.doc.clientID]
+                        )
+                    );
+                    websocket?.send(
+                        encoding.toUint8Array(encoderAwarenessState)
+                    );
+                }
+            };
+
+            provider.emit('status', [{ status: 'connecting' }]);
+        })
+        .catch(err => {
+            if (reconnect <= 0) {
+                provider.emit('lost-connection', []);
+            }
+            provider.wsUnsuccessfulReconnects++;
+            setTimeout(
+                registerWebsocket,
+                math.min(
+                    math.pow(2, provider.wsUnsuccessfulReconnects) * 100,
+                    provider.maxBackOffTime
+                ),
+                provider,
+                token,
+                resyncInterval,
+                reconnect > 0 ? reconnect - 1 : 3
+            );
+        });
+
+    let resyncInterval = 0;
+    if (resync > 0) {
+        resyncInterval = setInterval(() => {
+            if (websocket?.readyState === WebSocket.OPEN) {
+                // resend sync step 1
+                const encoder = encoding.createEncoder();
+                encoding.writeVarUint(encoder, Message.sync);
+                syncProtocol.writeSyncStep1(encoder, provider.doc);
+                websocket.send(encoding.toUint8Array(encoder));
+            }
+        }, resync) as unknown as number;
+    }
+
+    const checkInterval = setInterval(() => {
+        if (
+            state === WebSocketState.connected &&
+            WEBSOCKET_RECONNECT < time.getUnixTime() - lastMessageReceived
+        ) {
+            // no message received in a long time - not even your own awareness
+            // updates (which are updated every 15 seconds)
+            websocket?.close();
+        }
+    }, WEBSOCKET_RECONNECT / 10);
+
+    const broadcastMessage = (buf: ArrayBuffer) => {
+        if (state === WebSocketState.connected) {
+            websocket?.send(buf);
+        }
+    };
+
+    const disconnect = () => {
+        if (websocket != null) {
+            websocket.close();
+            websocket = undefined;
+            state = WebSocketState.disconnected;
+            if (resyncInterval !== 0) {
+                clearInterval(resyncInterval);
+            }
+            clearInterval(checkInterval);
+        }
+    };
+
+    return { broadcastMessage, disconnect };
+};

libs/datasource/jwt-rpc/src/handler.ts

@@ -0,0 +1,82 @@
+import * as encoding from 'lib0/encoding';
+import * as decoding from 'lib0/decoding';
+import * as authProtocol from 'y-protocols/auth';
+import * as awarenessProtocol from 'y-protocols/awareness';
+import * as syncProtocol from 'y-protocols/sync';
+
+import { WebsocketProvider } from './provider';
+
+const permissionDeniedHandler = (provider: WebsocketProvider, reason: string) =>
+    console.warn(`Permission denied to access ${provider.url}.\n${reason}`);
+
+export enum Message {
+    sync = 0,
+    queryAwareness,
+    awareness,
+    auth,
+}
+
+export type MessageCallback = (
+    encoder: encoding.Encoder,
+    decoder: decoding.Decoder,
+    provider: WebsocketProvider,
+    emitSynced: boolean,
+    messageType: number
+) => void;
+
+export const handler: Record<Message, MessageCallback> = {
+    [Message.sync]: (encoder, decoder, provider, emitSynced, messageType) => {
+        encoding.writeVarUint(encoder, Message.sync);
+        const syncMessageType = syncProtocol.readSyncMessage(
+            decoder,
+            encoder,
+            provider.doc,
+            provider
+        );
+        if (
+            emitSynced &&
+            syncMessageType === syncProtocol.messageYjsSyncStep2 &&
+            !provider.synced
+        ) {
+            provider.synced = true;
+        }
+    },
+    [Message.queryAwareness]: (
+        encoder,
+        decoder,
+        provider,
+        emitSynced,
+        messageType
+    ) => {
+        encoding.writeVarUint(encoder, Message.queryAwareness);
+        encoding.writeVarUint8Array(
+            encoder,
+            awarenessProtocol.encodeAwarenessUpdate(
+                provider.awareness,
+                Array.from(provider.awareness.getStates().keys())
+            )
+        );
+    },
+
+    [Message.awareness]: (
+        encoder,
+        decoder,
+        provider,
+        emitSynced,
+        messageType
+    ) => {
+        awarenessProtocol.applyAwarenessUpdate(
+            provider.awareness,
+            decoding.readVarUint8Array(decoder),
+            provider
+        );
+    },
+
+    [Message.auth]: (encoder, decoder, provider, emitSynced, messageType) => {
+        authProtocol.readAuthMessage(
+            decoder,
+            provider.doc,
+            permissionDeniedHandler
+        );
+    },
+};

libs/datasource/jwt-rpc/src/index.ts

@@ -0,0 +1 @@
+export { WebsocketProvider } from './provider';

libs/datasource/jwt-rpc/src/provider.ts

@@ -0,0 +1,134 @@
+import * as Y from 'yjs';
+
+import { Observable } from 'lib0/observable';
+import * as url from 'lib0/url';
+
+import * as awarenessProtocol from 'y-protocols/awareness';
+
+import { handler } from './handler';
+import {
+    registerBroadcastSubscriber,
+    registerUpdateHandler,
+    registerWebsocket,
+} from './connector';
+
+/**
+ * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document.
+ * The document name is attached to the provided url. I.e. the following example
+ * creates a websocket connection to http://localhost:3000/my-document-name
+ *
+ * @example
+ *   import * as Y from 'yjs'
+ *   import { WebsocketProvider } from 'jwt-rpc'
+ *   const doc = new Y.Doc()
+ *   const provider = new WebsocketProvider('http://localhost:3000', 'my-document-name', doc)
+ */
+export class WebsocketProvider extends Observable<string> {
+    maxBackOffTime: number;
+    url: string;
+    roomName: string;
+
+    awareness: awarenessProtocol.Awareness;
+    doc: Y.Doc;
+
+    wsUnsuccessfulReconnects: number;
+    private _synced: boolean;
+
+    broadcastChannel: string;
+    private _broadcast?: {
+        broadcastMessage: (buf: ArrayBuffer) => void;
+        disconnect: () => void;
+    };
+
+    private _websocket?: {
+        broadcastMessage: (buf: ArrayBuffer) => void;
+        disconnect: () => void;
+    };
+
+    private _updateHandlerDestroy: () => void;
+
+    constructor(
+        token: string,
+        serverUrl: string,
+        roomName: string,
+        doc: Y.Doc,
+        {
+            awareness = new awarenessProtocol.Awareness(doc),
+            params = {},
+            resyncInterval = -1,
+            maxBackOffTime = 2500,
+        } = {}
+    ) {
+        super();
+
+        this.roomName = roomName;
+        // ensure that url is always ends with /
+        while (serverUrl[serverUrl.length - 1] === '/') {
+            serverUrl = serverUrl.slice(0, serverUrl.length - 1);
+        }
+        this.broadcastChannel = serverUrl + '/' + roomName;
+        const encodedParams = url.encodeQueryParams(params);
+        this.url =
+            this.broadcastChannel +
+            (encodedParams.length === 0 ? '' : '?' + encodedParams);
+
+        this.awareness = awareness;
+        this.doc = doc;
+
+        this.maxBackOffTime = maxBackOffTime;
+        this.wsUnsuccessfulReconnects = 0;
+
+        this._synced = false;
+
+        this._websocket = registerWebsocket(this, token, resyncInterval);
+        // TODO: The current implementation will cause a broadcast storm, and then re-implement one, or just go websocket synchronization
+        // this._broadcast = registerBroadcastSubscriber(
+        //     this,
+        //     this.awareness,
+        //     this.doc
+        // );
+
+        this._updateHandlerDestroy = registerUpdateHandler(
+            this,
+            awareness,
+            doc,
+            buf => {
+                this._websocket?.broadcastMessage(buf);
+                this._broadcast?.broadcastMessage(buf);
+            }
+        );
+    }
+
+    get messageHandlers() {
+        return handler;
+    }
+
+    get synced() {
+        return this._synced;
+    }
+
+    set synced(state) {
+        if (this._synced !== state) {
+            this._synced = state;
+            this.emit('synced', [state]);
+            this.emit('sync', [state]);
+        }
+    }
+
+    override destroy() {
+        if (this._broadcast) {
+            const disconnect = this._broadcast.disconnect;
+            this._broadcast = undefined;
+            disconnect();
+        }
+
+        if (this._websocket) {
+            const disconnect = this._websocket.disconnect;
+            this._websocket = undefined;
+            disconnect();
+        }
+
+        this._updateHandlerDestroy?.();
+        super.destroy();
+    }
+}

libs/datasource/jwt-rpc/tsconfig.json

@@ -0,0 +1,27 @@
+{
+    "extends": "../../../tsconfig.base.json",
+    "compilerOptions": {
+        "jsx": "react-jsx",
+        "allowJs": true,
+        "esModuleInterop": true,
+        "allowSyntheticDefaultImports": true,
+        "forceConsistentCasingInFileNames": true,
+        "strict": true,
+        "noImplicitOverride": true,
+        "noPropertyAccessFromIndexSignature": true,
+        "noImplicitReturns": true,
+        "noFallthroughCasesInSwitch": true,
+        "noImplicitAny": true,
+        "strictNullChecks": true
+    },
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.lib.json"
+        },
+        {
+            "path": "./tsconfig.spec.json"
+        }
+    ]
+}

libs/datasource/jwt-rpc/tsconfig.lib.json

@@ -0,0 +1,23 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "files": [
+        "../../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+        "../../../node_modules/@nrwl/react/typings/image.d.ts"
+    ],
+    "exclude": [
+        "jest.config.ts",
+        "**/*.spec.ts",
+        "**/*.test.ts",
+        "**/*.spec.tsx",
+        "**/*.test.tsx",
+        "**/*.spec.js",
+        "**/*.test.js",
+        "**/*.spec.jsx",
+        "**/*.test.jsx"
+    ],
+    "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]
+}

libs/datasource/jwt-rpc/tsconfig.spec.json

@@ -0,0 +1,20 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../../dist/out-tsc",
+        "module": "commonjs",
+        "types": ["jest", "node"]
+    },
+    "include": [
+        "jest.config.ts",
+        "**/*.test.ts",
+        "**/*.spec.ts",
+        "**/*.test.tsx",
+        "**/*.spec.tsx",
+        "**/*.test.js",
+        "**/*.spec.js",
+        "**/*.test.jsx",
+        "**/*.spec.jsx",
+        "**/*.d.ts"
+    ]
+}

libs/datasource/jwt/.babelrc

@@ -0,0 +1,12 @@
+{
+    "presets": [
+        [
+            "@nrwl/react/babel",
+            {
+                "runtime": "automatic",
+                "useBuiltIns": "usage"
+            }
+        ]
+    ],
+    "plugins": []
+}

libs/datasource/jwt/.eslintrc.json

@@ -0,0 +1,27 @@
+{
+    "extends": ["plugin:@nrwl/nx/react", "../../../.eslintrc.json"],
+    "ignorePatterns": ["!**/*"],
+    "overrides": [
+        {
+            "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+            "rules": {}
+        },
+        {
+            "files": ["*.ts", "*.tsx"],
+            "rules": {
+                "@typescript-eslint/no-unused-vars": [
+                    "error",
+                    {
+                        "vars": "all",
+                        "args": "after-used",
+                        "ignoreRestSiblings": false
+                    }
+                ]
+            }
+        },
+        {
+            "files": ["*.js", "*.jsx"],
+            "rules": {}
+        }
+    ]
+}

libs/datasource/jwt/jest.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+    displayName: 'datasource-jwt',
+    preset: '../../../jest.preset.js',
+    transform: {
+        '^.+\\.[tj]sx?$': 'babel-jest',
+    },
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
+    coverageDirectory: '../../../coverage/libs/datasource/jwt',
+};

libs/datasource/jwt/package.json

@@ -0,0 +1,33 @@
+{
+    "name": "@toeverything/datasource/jwt",
+    "version": "0.0.1",
+    "license": "MIT",
+    "author": "DarkSky <darksky2048@gmail.com>",
+    "devDependencies": {
+        "@types/debug": "^4.1.7",
+        "@types/file-saver": "^2.0.5",
+        "@types/uuid": "^8.3.4",
+        "@types/wicg-file-system-access": "^2020.9.5",
+        "file-saver": "^2.0.5",
+        "file-selector": "^0.6.0",
+        "flexsearch": "^0.7.21",
+        "lib0": "^0.2.51",
+        "lru-cache": "^7.12.0",
+        "ts-debounce": "^4.0.0",
+        "y-indexeddb": "^9.0.8"
+    },
+    "dependencies": {
+        "@types/flexsearch": "^0.7.3",
+        "buffer": "^6.0.3",
+        "debug": "^4.3.4",
+        "fflate": "^0.7.3",
+        "idb-keyval": "^6.2.0",
+        "immer": "^9.0.15",
+        "nanoid": "^4.0.0",
+        "sha3": "^2.1.4",
+        "sift": "^16.0.0",
+        "uuid": "^8.3.2",
+        "y-protocols": "^1.0.5",
+        "yjs": "^13.5.39"
+    }
+}

libs/datasource/jwt/project.json

@@ -0,0 +1,43 @@
+{
+    "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "libs/datasource/jwt/src",
+    "projectType": "library",
+    "tags": ["datasource:jwt"],
+    "targets": {
+        "build": {
+            "executor": "@nrwl/web:rollup",
+            "outputs": ["{options.outputPath}"],
+            "options": {
+                "outputPath": "dist/libs/datasource/jwt",
+                "tsConfig": "libs/datasource/jwt/tsconfig.lib.json",
+                "project": "libs/datasource/jwt/package.json",
+                "entryFile": "libs/datasource/jwt/src/index.ts",
+                "external": ["react/jsx-runtime"],
+                "rollupConfig": "@nrwl/react/plugins/bundle-rollup",
+                "compiler": "babel",
+                "assets": [
+                    {
+                        "glob": "libs/datasource/jwt/README.md",
+                        "input": ".",
+                        "output": "."
+                    }
+                ]
+            }
+        },
+        "lint": {
+            "executor": "@nrwl/linter:eslint",
+            "outputs": ["{options.outputFile}"],
+            "options": {
+                "lintFilePatterns": ["libs/datasource/jwt/**/*.{ts,tsx,js,jsx}"]
+            }
+        },
+        "test": {
+            "executor": "@nrwl/jest:jest",
+            "outputs": ["coverage/libs/datasource/jwt"],
+            "options": {
+                "jestConfig": "libs/datasource/jwt/jest.config.js",
+                "passWithNoTests": true
+            }
+        }
+    }
+}

libs/datasource/jwt/src/adapter/index.ts

@@ -0,0 +1,161 @@
+import { AbstractType as YAbstractType } from 'yjs';
+
+import { BlockItem } from '../types';
+
+export type ChangedStateKeys = 'add' | 'update' | 'delete';
+export type ChangedStates<S = ChangedStateKeys> = Map<string, S>;
+
+export type BlockListener<S = ChangedStateKeys, R = unknown> = (
+    states: ChangedStates<S>
+) => Promise<R> | R;
+
+export type Operable<T, Base = YAbstractType<any>> = T extends Base
+    ? ContentOperation
+    : T;
+
+export interface InternalPlainObject {}
+
+export type BaseTypes = string | number | boolean | InternalPlainObject;
+
+export type ContentTypes = BaseTypes | ContentOperation;
+
+interface ContentOperation {
+    get length(): number;
+    createText(): TextOperation;
+    createArray<T extends ContentTypes = ContentOperation>(): ArrayOperation<T>;
+    createMap<T extends ContentTypes = ContentOperation>(): MapOperation<T>;
+    asText(): TextOperation | undefined;
+    asArray<T extends ContentTypes = ContentOperation>():
+        | ArrayOperation<T>
+        | undefined;
+    asMap<T extends ContentTypes = ContentOperation>():
+        | MapOperation<T>
+        | undefined;
+    autoGet(
+        root: ThisType<ContentOperation> | Record<string, unknown>,
+        path: string[]
+    ): unknown | undefined;
+    autoSet(
+        root: ThisType<ContentOperation>,
+        path: string[],
+        data: unknown,
+        partial?: boolean
+    ): void;
+}
+
+type TextAttributes = Record<string, string>;
+
+export type TextToken = {
+    insert: string;
+    attributes?: TextAttributes;
+};
+
+export interface TextOperation extends ContentOperation {
+    insert(
+        index: number,
+        content: string,
+        format?: Record<string, string>
+    ): void;
+    format(index: number, length: number, format: Record<string, string>): void;
+    delete(index: number, length: number): void;
+    setAttribute(name: string, value: BaseTypes): void;
+    getAttribute<T extends BaseTypes = string>(name: string): T | undefined;
+    toString(): TextToken[];
+}
+
+export interface ArrayOperation<T extends ContentTypes = ContentOperation>
+    extends ContentOperation {
+    insert(index: number, content: Array<Operable<T>>): void;
+    delete(index: number, length: number): void;
+    push(content: Array<Operable<T>>): void;
+    unshift(content: Array<Operable<T>>): void;
+    get(index: number): Operable<T> | undefined;
+    slice(start?: number, end?: number): Array<Operable<T>>;
+    map<R = unknown>(callback: (value: T, index: number) => R): Array<R>;
+    forEach(callback: (value: T, index: number) => boolean | void): void;
+    find<R = unknown>(
+        callback: (value: T, index: number) => boolean
+    ): R | undefined;
+    findIndex(callback: (value: T, index: number) => boolean): number;
+}
+
+export interface MapOperation<T extends ContentTypes = ContentOperation>
+    extends ContentOperation {
+    set(key: string, value: Operable<T>): void;
+    get(key: string): Operable<T> | undefined;
+    delete(key: string): void;
+    has(key: string): boolean;
+}
+
+export type HistoryCallback<T = unknown> = (map: Map<string, T>) => void;
+
+export interface HistoryManager {
+    onPush<T = unknown>(name: string, callback: HistoryCallback<T>): void;
+    offPush(name: string): boolean;
+    onPop<T = unknown>(name: string, callback: HistoryCallback<T>): void;
+    offPop(name: string): boolean;
+    undo<T = unknown>(): Map<string, T> | undefined;
+    redo<T = unknown>(): Map<string, T> | undefined;
+    clear(): void;
+}
+
+type BlockPosition = { pos?: number; before?: string; after?: string };
+
+interface BlockInstance<C extends ContentOperation> {
+    get id(): string;
+    get type(): BlockItem<C>['type'];
+    get flavor(): BlockItem<C>['flavor'];
+    // TODO: flavor needs optimization
+    setFlavor(flavor: BlockItem<C>['flavor']): void;
+    get created(): BlockItem<C>['created'];
+    get updated(): number; // update time, UTC timestamp, read only
+    get creator(): string | undefined; // creator id
+    get children(): string[];
+    getChildren(block_ids: (string | undefined)[]): BlockInstance<C>[];
+    hasChildren(block_id: string): boolean;
+    insertChildren(
+        block: ThisType<BlockInstance<C>>,
+        pos?: BlockPosition
+    ): void;
+    removeChildren(block_ids: (string | undefined)[]): void;
+    get content(): BlockItem<C>['content'];
+    on(
+        key: 'children' | 'content',
+        name: string,
+        listener: BlockListener
+    ): void;
+    off(key: 'children' | 'content', name: string): void;
+    addChildrenListener(name: string, listener: BlockListener): void;
+    removeChildrenListener(name: string): void;
+    addContentListener(name: string, listener: BlockListener): void;
+    removeContentListener(name: string): void;
+    scopedHistory(scope: any[]): HistoryManager;
+}
+
+interface AsyncDatabaseAdapter<C extends ContentOperation> {
+    inspector(): Record<string, any>;
+    createBlock(
+        options: Pick<BlockItem<C>, 'type' | 'flavor'> & {
+            binary?: ArrayBuffer;
+            uuid?: string;
+        }
+    ): Promise<BlockInstance<C>>;
+    getBlock(id: string): Promise<BlockInstance<C> | undefined>;
+    getBlockByFlavor(flavor: BlockItem<C>['flavor']): Promise<string[]>;
+    getBlockByType(type: BlockItem<C>['type']): Promise<string[]>;
+    checkBlocks(keys: string[]): Promise<boolean>;
+    deleteBlocks(keys: string[]): Promise<string[]>;
+    on<S, R>(key: 'editing' | 'updated', listener: BlockListener<S, R>): void;
+    suspend(suspend: boolean): void;
+    history(): HistoryManager;
+    getUserId(): string;
+}
+
+export type {
+    AsyncDatabaseAdapter,
+    BlockPosition,
+    BlockInstance,
+    ContentOperation,
+};
+export type { YjsInitOptions, YjsContentOperation } from './yjs';
+export { YjsAdapter } from './yjs';

libs/datasource/jwt/src/adapter/yjs/binary.ts

@@ -0,0 +1,66 @@
+import { Array as YArray, Map as YMap } from 'yjs';
+
+import { RemoteKvService } from '@toeverything/datasource/remote-kv';
+
+export class YjsRemoteBinaries {
+    readonly #binaries: YMap<YArray<ArrayBuffer>>; // binary instance
+    readonly #remote_storage?: RemoteKvService;
+
+    constructor(binaries: YMap<YArray<ArrayBuffer>>, remote_token?: string) {
+        this.#binaries = binaries;
+        if (remote_token) {
+            this.#remote_storage = new RemoteKvService(remote_token);
+        } else {
+            console.warn(`Remote storage is not ready`);
+        }
+    }
+
+    has(name: string): boolean {
+        return this.#binaries.has(name);
+    }
+
+    async get(name: string): Promise<YArray<ArrayBuffer> | undefined> {
+        if (this.#binaries.has(name)) {
+            return this.#binaries.get(name);
+        } else {
+            // TODO: Remote Load
+            try {
+                const file = await this.#remote_storage?.instance.getBuffData(
+                    name
+                );
+                console.log(file);
+                // return file;
+            } catch (e) {
+                throw new Error(`Binary ${name} not found`);
+            }
+            return undefined;
+        }
+    }
+
+    async set(name: string, binary: YArray<ArrayBuffer>) {
+        if (!this.#binaries.has(name)) {
+            console.log(name, 'name');
+            if (binary.length === 1) {
+                this.#binaries.set(name, binary);
+                if (this.#remote_storage) {
+                    // TODO: Remote Save, if there is an object with the same name remotely, the upload is skipped, because the file name is the hash of the file content
+                    const has_file = this.#remote_storage.instance.exist(name);
+                    if (!has_file) {
+                        const upload_file = new File(binary.toArray(), name);
+                        await this.#remote_storage.instance
+                            .upload(upload_file)
+                            .catch(err => {
+                                throw new Error(`${err} upload error`);
+                            });
+                    }
+                } else {
+                    console.warn(`Remote storage is not ready`);
+                }
+                return;
+            } else {
+                console.log('err');
+            }
+            throw new Error(`Binary ${name} is invalid`);
+        }
+    }
+}

libs/datasource/jwt/src/adapter/yjs/block.ts

@@ -0,0 +1,293 @@
+import {
+    AbstractType as YAbstractType,
+    Array as YArray,
+    Map as YMap,
+    transact,
+} from 'yjs';
+
+import { BlockInstance, BlockListener, HistoryManager } from '../index';
+import { BlockItem, BlockTypes } from '../../types';
+
+import { YjsContentOperation } from './operation';
+import { ChildrenListenerHandler, ContentListenerHandler } from './listener';
+import { YjsHistoryManager } from './history';
+
+const GET_BLOCK_ITEM = Symbol('GET_BLOCK_ITEM');
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+const getMapFromYArray = (array: YArray<string>) =>
+    new Map(array.map((child, index) => [child, index]));
+
+type YjsBlockInstanceProps = {
+    id: string;
+    block: YMap<unknown>;
+    binary?: YArray<ArrayBuffer>;
+    setBlock: (
+        id: string,
+        block: BlockItem<YjsContentOperation>
+    ) => Promise<void>;
+    getUpdated: (id: string) => number | undefined;
+    getCreator: (id: string) => string | undefined;
+    getBlockInstance: (id: string) => YjsBlockInstance | undefined;
+};
+
+export class YjsBlockInstance implements BlockInstance<YjsContentOperation> {
+    readonly #id: string;
+    readonly #block: YMap<unknown>;
+    readonly #binary?: YArray<ArrayBuffer>;
+    readonly #children: YArray<string>;
+    readonly #set_block: (
+        id: string,
+        block: BlockItem<YjsContentOperation>
+    ) => Promise<void>;
+    readonly #get_updated: (id: string) => number | undefined;
+    readonly #get_creator: (id: string) => string | undefined;
+    readonly #get_block_instance: (id: string) => YjsBlockInstance | undefined;
+    readonly #children_listeners: Map<string, BlockListener>;
+    readonly #content_listeners: Map<string, BlockListener>;
+
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    #children_map: Map<string, number>;
+
+    constructor(props: YjsBlockInstanceProps) {
+        this.#id = props.id;
+        this.#block = props.block;
+        this.#binary = props.binary;
+
+        this.#children = props.block.get('children') as YArray<string>;
+        this.#children_map = getMapFromYArray(this.#children);
+        this.#set_block = props.setBlock;
+        this.#get_updated = props.getUpdated;
+        this.#get_creator = props.getCreator;
+        this.#get_block_instance = props.getBlockInstance;
+
+        this.#children_listeners = new Map();
+        this.#content_listeners = new Map();
+
+        const content = this.#block.get('content') as YMap<unknown>;
+
+        this.#children.observe(event =>
+            ChildrenListenerHandler(this.#children_listeners, event)
+        );
+        content?.observeDeep(events =>
+            ContentListenerHandler(this.#content_listeners, events)
+        );
+        // TODO: flavor needs optimization
+        this.#block.observeDeep(events =>
+            ContentListenerHandler(this.#content_listeners, events)
+        );
+    }
+
+    on(
+        key: 'children' | 'content',
+        name: string,
+        listener: BlockListener
+    ): void {
+        if (key === 'children') {
+            this.addChildrenListener(name, listener);
+        } else if (key === 'content') {
+            this.addContentListener(name, listener);
+        }
+    }
+
+    off(key: 'children' | 'content', name: string): void {
+        if (key === 'children') {
+            this.removeChildrenListener(name);
+        } else if (key === 'content') {
+            this.removeContentListener(name);
+        }
+    }
+
+    addChildrenListener(name: string, listener: BlockListener): void {
+        this.#children_listeners.set(name, listener);
+    }
+
+    removeChildrenListener(name: string): void {
+        this.#children_listeners.delete(name);
+    }
+
+    addContentListener(name: string, listener: BlockListener): void {
+        this.#content_listeners.set(name, listener);
+    }
+
+    removeContentListener(name: string): void {
+        this.#content_listeners.delete(name);
+    }
+
+    get id() {
+        return this.#id;
+    }
+
+    get content(): YjsContentOperation {
+        if (this.type === BlockTypes.block) {
+            const content = this.#block.get('content');
+            if (content instanceof YAbstractType) {
+                return new YjsContentOperation(content);
+            } else {
+                throw new Error(`Invalid content type: ${typeof content}`);
+            }
+        } else if (this.type === BlockTypes.binary && this.#binary) {
+            return new YjsContentOperation(this.#binary);
+        }
+        throw new Error(
+            `Invalid content type: ${this.type}, ${this.#block.get(
+                'content'
+            )}, ${this.#binary}`
+        );
+    }
+
+    get type(): BlockItem<YjsContentOperation>['type'] {
+        return this.#block.get(
+            'type'
+        ) as BlockItem<YjsContentOperation>['type'];
+    }
+
+    get flavor(): BlockItem<YjsContentOperation>['flavor'] {
+        return this.#block.get(
+            'flavor'
+        ) as BlockItem<YjsContentOperation>['flavor'];
+    }
+
+    // TODO: bad case. Need to optimize.
+    setFlavor(flavor: BlockItem<YjsContentOperation>['flavor']) {
+        this.#block.set('flavor', flavor);
+    }
+
+    get created(): BlockItem<YjsContentOperation>['created'] {
+        return this.#block.get(
+            'created'
+        ) as BlockItem<YjsContentOperation>['created'];
+    }
+
+    get updated(): number {
+        return this.#get_updated(this.#id) || this.created;
+    }
+
+    get creator(): string | undefined {
+        return this.#get_creator(this.#id);
+    }
+
+    get children(): string[] {
+        return this.#children.toArray();
+    }
+
+    getChildren(ids?: (string | undefined)[]): YjsBlockInstance[] {
+        const query_ids = ids?.filter((id): id is string => !!id) || [];
+        const exists_ids = this.#children.map(id => id);
+        const filter_ids = query_ids.length ? query_ids : exists_ids;
+        return exists_ids
+            .filter(id => filter_ids.includes(id))
+            .map(id => this.#get_block_instance(id))
+            .filter((v): v is YjsBlockInstance => !!v);
+    }
+
+    hasChildren(id: string): boolean {
+        if (this.children.includes(id)) return true;
+        return this.getChildren().some(block => block.hasChildren(id));
+    }
+
+    private position_calculator(
+        max_pos: number,
+        position?: { pos?: number; before?: string; after?: string }
+    ) {
+        const { pos, before, after } = position || {};
+        if (typeof pos === 'number' && Number.isInteger(pos)) {
+            if (pos >= 0 && pos < max_pos) {
+                return pos;
+            }
+        } else if (before) {
+            const current_pos = this.#children_map.get(before || '');
+            if (
+                typeof current_pos === 'number' &&
+                Number.isInteger(current_pos)
+            ) {
+                const prev_pos = current_pos;
+                if (prev_pos >= 0 && prev_pos < max_pos) {
+                    return prev_pos;
+                }
+            }
+        } else if (after) {
+            const current_pos = this.#children_map.get(after || '');
+            if (
+                typeof current_pos === 'number' &&
+                Number.isInteger(current_pos)
+            ) {
+                const next_pos = current_pos + 1;
+                if (next_pos >= 0 && next_pos < max_pos) {
+                    return next_pos;
+                }
+            }
+        }
+        return undefined;
+    }
+
+    async insertChildren(
+        block: YjsBlockInstance,
+        pos?: { pos?: number; before?: string; after?: string }
+    ): Promise<void> {
+        const content = block[GET_BLOCK_ITEM]();
+        if (content) {
+            const lastIndex = this.#children_map.get(block.id);
+            if (typeof lastIndex === 'number') {
+                this.#children.delete(lastIndex);
+                this.#children_map = getMapFromYArray(this.#children);
+            }
+
+            const position = this.position_calculator(
+                this.#children_map.size,
+                pos
+            );
+            if (typeof position === 'number') {
+                this.#children.insert(position, [block.id]);
+            } else {
+                this.#children.push([block.id]);
+            }
+            await this.#set_block(block.id, content);
+            this.#children_map = getMapFromYArray(this.#children);
+        }
+    }
+
+    removeChildren(ids: (string | undefined)[]): Promise<string[]> {
+        return new Promise(resolve => {
+            if (this.#children.doc) {
+                transact(this.#children.doc, () => {
+                    const failed = [];
+                    for (const id of ids) {
+                        let idx = -1;
+                        for (const block_id of this.#children) {
+                            idx += 1;
+                            if (block_id === id) {
+                                this.#children.delete(idx);
+                                break;
+                            }
+                        }
+                        if (id) failed.push(id);
+                    }
+
+                    this.#children_map = getMapFromYArray(this.#children);
+                    resolve(failed);
+                });
+            } else {
+                resolve(ids.filter((id): id is string => !!id));
+            }
+        });
+    }
+
+    public scopedHistory(scope: any[]): HistoryManager {
+        return new YjsHistoryManager(this.#block, scope);
+    }
+
+    [GET_BLOCK_ITEM]() {
+        // check null & undefined
+        if (this.content != null) {
+            return {
+                type: this.type,
+                flavor: this.flavor,
+                children: this.#children.slice(),
+                created: this.created,
+                content: this.content,
+            };
+        }
+        return undefined;
+    }
+}

libs/datasource/jwt/src/adapter/yjs/gatekeeper.ts

@@ -0,0 +1,53 @@
+import { Map as YMap } from 'yjs';
+
+export class GateKeeper {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    #user_id: string;
+    #creators: YMap<string>;
+    #common: YMap<string>;
+
+    constructor(userId: string, creators: YMap<string>, common: YMap<string>) {
+        this.#user_id = userId;
+        this.#creators = creators;
+        this.#common = common;
+    }
+
+    getCreator(block_id: string): string | undefined {
+        return this.#creators.get(block_id) || this.#common.get(block_id);
+    }
+
+    setCreator(block_id: string) {
+        if (!this.#creators.get(block_id)) {
+            this.#creators.set(block_id, this.#user_id);
+        }
+    }
+
+    setCommon(block_id: string) {
+        if (!this.#creators.get(block_id) && !this.#common.get(block_id)) {
+            this.#common.set(block_id, this.#user_id);
+        }
+    }
+
+    private check_delete(block_id: string): boolean {
+        const creator = this.#creators.get(block_id);
+        return creator === this.#user_id || !!this.#common.get(block_id);
+    }
+
+    checkDeleteLists(block_ids: string[]) {
+        const success = [];
+        const fail = [];
+        for (const block_id of block_ids) {
+            if (this.check_delete(block_id)) {
+                success.push(block_id);
+            } else {
+                fail.push(block_id);
+            }
+        }
+        return [success, fail];
+    }
+
+    clear() {
+        this.#creators.clear();
+        this.#common.clear();
+    }
+}

libs/datasource/jwt/src/adapter/yjs/history.ts

@@ -0,0 +1,73 @@
+import { Map as YMap, UndoManager } from 'yjs';
+
+import { HistoryCallback, HistoryManager } from '../../adapter';
+
+type StackItem = UndoManager['undoStack'][0];
+
+export class YjsHistoryManager implements HistoryManager {
+    readonly #blocks: YMap<any>;
+    readonly #history_manager: UndoManager;
+    readonly #push_listeners: Map<string, HistoryCallback<any>>;
+    readonly #pop_listeners: Map<string, HistoryCallback<any>>;
+
+    constructor(scope: YMap<any>, tracker?: any[]) {
+        this.#blocks = scope;
+        this.#history_manager = new UndoManager(scope, {
+            trackedOrigins: tracker ? new Set(tracker) : undefined,
+        });
+
+        this.#push_listeners = new Map();
+        this.#history_manager.on(
+            'stack-item-added',
+            (event: { stackItem: StackItem }) => {
+                const meta = event.stackItem.meta;
+                for (const listener of this.#push_listeners.values()) {
+                    listener(meta);
+                }
+            }
+        );
+
+        this.#pop_listeners = new Map();
+        this.#history_manager.on(
+            'stack-item-popped',
+            (event: { stackItem: StackItem }) => {
+                const meta = event.stackItem.meta;
+                for (const listener of this.#pop_listeners.values()) {
+                    listener(new Map(meta));
+                }
+            }
+        );
+    }
+
+    onPush<T = unknown>(name: string, callback: HistoryCallback<T>): void {
+        this.#push_listeners.set(name, callback);
+    }
+
+    offPush(name: string): boolean {
+        return this.#push_listeners.delete(name);
+    }
+
+    onPop<T = unknown>(name: string, callback: HistoryCallback<T>): void {
+        this.#pop_listeners.set(name, callback);
+    }
+
+    offPop(name: string): boolean {
+        return this.#pop_listeners.delete(name);
+    }
+
+    break(): void {
+        // this.#history_manager.
+    }
+
+    undo<T = unknown>(): Map<string, T> | undefined {
+        return this.#history_manager.undo()?.meta;
+    }
+
+    redo<T = unknown>(): Map<string, T> | undefined {
+        return this.#history_manager.redo()?.meta;
+    }
+
+    clear(): void {
+        return this.#history_manager.clear();
+    }
+}

libs/datasource/jwt/src/adapter/yjs/index.ts

@@ -0,0 +1,605 @@
+/* eslint-disable max-lines */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/// <reference types="wicg-file-system-access" />
+import { Buffer } from 'buffer';
+import { saveAs } from 'file-saver';
+import { fromEvent } from 'file-selector';
+import LRUCache from 'lru-cache';
+import { debounce } from 'ts-debounce';
+import { nanoid } from 'nanoid';
+import { IndexeddbPersistence } from 'y-indexeddb';
+import { Awareness } from 'y-protocols/awareness.js';
+import {
+    Doc,
+    Array as YArray,
+    Map as YMap,
+    transact,
+    encodeStateAsUpdate,
+    applyUpdate,
+} from 'yjs';
+
+import { WebsocketProvider } from '@toeverything/datasource/jwt-rpc';
+
+import {
+    AsyncDatabaseAdapter,
+    BlockListener,
+    ChangedStateKeys,
+    HistoryManager,
+} from '../../adapter';
+import { BucketBackend, BlockItem, BlockTypes } from '../../types';
+import { getLogger, sha3, sleep } from '../../utils';
+
+import { YjsRemoteBinaries } from './binary';
+import { YjsBlockInstance } from './block';
+import { GateKeeper } from './gatekeeper';
+import {
+    YjsContentOperation,
+    DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_SYMBOL_INTO_INNER as INTO_INNER,
+} from './operation';
+import { EmitEvents, Suspend } from './listener';
+import { YjsHistoryManager } from './history';
+
+declare const JWT_DEV: boolean;
+const logger = getLogger('BlockDB:yjs');
+
+type YjsProviders = {
+    awareness: Awareness;
+    idb: IndexeddbPersistence;
+    binariesIdb: IndexeddbPersistence;
+    ws?: WebsocketProvider;
+    backend: string;
+    gatekeeper: GateKeeper;
+    userId: string;
+    remoteToken?: string; // remote storage token
+};
+const _yjsDatabaseInstance = new Map<string, YjsProviders>();
+
+async function _initWebsocketProvider(
+    url: string,
+    room: string,
+    doc: Doc,
+    token?: string,
+    params?: YjsInitOptions['params']
+): Promise<[Awareness, WebsocketProvider | undefined]> {
+    const awareness = new Awareness(doc);
+    if (token && !process.env['NX_FREE_LOGIN']) {
+        const ws = new WebsocketProvider(token, url, room, doc, {
+            awareness,
+            params,
+        }) as any; // TODO: type is erased after cascading references
+
+        // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
+        return new Promise((resolve, reject) => {
+            // TODO: synced will also be triggered on reconnection after losing sync
+            // There needs to be an event mechanism to emit the synchronization state to the upper layer
+            ws.once('synced', () => resolve([awareness, ws]));
+            ws.once('lost-connection', () => resolve([awareness, ws]));
+            ws.on('connection-error', reject);
+        });
+    } else {
+        return [awareness, undefined];
+    }
+}
+
+const _asyncInitLoading = new Set<string>();
+const _waitLoading = async (workspace: string) => {
+    while (_asyncInitLoading.has(workspace)) {
+        await sleep();
+    }
+};
+
+async function _initYjsDatabase(
+    backend: string,
+    workspace: string,
+    options: {
+        params: YjsInitOptions['params'];
+        userId: string;
+        token?: string;
+    }
+): Promise<YjsProviders> {
+    if (_asyncInitLoading.has(workspace)) {
+        await _waitLoading(workspace);
+    }
+    const instance = _yjsDatabaseInstance.get(workspace);
+    // tTODO:odo temporarily handle this
+    if (
+        instance &&
+        (instance.userId === options.userId || options.userId === 'default')
+    ) {
+        return instance;
+    }
+    // if (instance) return instance;
+    _asyncInitLoading.add(workspace);
+    const { params, userId, token: remoteToken } = options;
+
+    const doc = new Doc({ autoLoad: true, shouldLoad: true });
+
+    const idb = await new IndexeddbPersistence(workspace, doc).whenSynced;
+    const [awareness, ws] = await _initWebsocketProvider(
+        backend,
+        workspace,
+        doc,
+        remoteToken,
+        params
+    );
+
+    const binaries = new Doc({ autoLoad: true, shouldLoad: true });
+    const binariesIdb = await new IndexeddbPersistence(
+        `${workspace}_binaries`,
+        binaries
+    ).whenSynced;
+
+    const gatekeeper = new GateKeeper(
+        userId,
+        doc.getMap('creators'),
+        doc.getMap('common')
+    );
+
+    _yjsDatabaseInstance.set(workspace, {
+        awareness,
+        idb,
+        binariesIdb,
+        ws,
+        backend,
+        gatekeeper,
+        userId,
+        remoteToken,
+    });
+    _asyncInitLoading.delete(workspace);
+
+    return {
+        awareness,
+        idb,
+        binariesIdb,
+        ws,
+        backend,
+        gatekeeper,
+        userId,
+        remoteToken,
+    };
+}
+
+export type { YjsBlockInstance } from './block';
+export type { YjsContentOperation } from './operation';
+
+export type YjsInitOptions = {
+    backend: typeof BucketBackend[keyof typeof BucketBackend];
+    params?: Record<string, string>;
+    userId?: string;
+    token?: string;
+};
+
+export class YjsAdapter implements AsyncDatabaseAdapter<YjsContentOperation> {
+    readonly #provider: YjsProviders;
+    readonly #doc: Doc; // doc instance
+    readonly #awareness: Awareness; // lightweight state synchronization
+    readonly #gatekeeper: GateKeeper; // Simple access control
+    readonly #history: YjsHistoryManager;
+
+    // Block Collection
+    // key is a randomly generated global id
+    readonly #blocks: YMap<YMap<unknown>>;
+    readonly #block_updated: YMap<number>;
+    // Maximum cache Block 1024, ttl 10 minutes
+    readonly #block_caches: LRUCache<string, YjsBlockInstance>;
+
+    readonly #binaries: YjsRemoteBinaries;
+
+    readonly #listener: Map<string, BlockListener<any>>;
+
+    static async init(
+        workspace: string,
+        options: YjsInitOptions
+    ): Promise<YjsAdapter> {
+        const { backend, params = {}, userId = 'default', token } = options;
+        const providers = await _initYjsDatabase(backend, workspace, {
+            params,
+            userId,
+            token,
+        });
+        return new YjsAdapter(providers);
+    }
+
+    private constructor(providers: YjsProviders) {
+        this.#provider = providers;
+        this.#doc = providers.idb.doc;
+        this.#awareness = providers.awareness;
+        this.#gatekeeper = providers.gatekeeper;
+
+        this.#blocks = this.#doc.getMap('blocks');
+        this.#block_updated = this.#doc.getMap('block_updated');
+        this.#block_caches = new LRUCache({ max: 1024, ttl: 1000 * 60 * 10 });
+        this.#binaries = new YjsRemoteBinaries(
+            providers.binariesIdb.doc.getMap(),
+            providers.remoteToken
+        );
+        this.#history = new YjsHistoryManager(this.#blocks);
+
+        this.#listener = new Map();
+
+        const debounced_editing_notifier = debounce(
+            () => {
+                const listener: BlockListener<Set<string>> | undefined =
+                    this.#listener.get('editing');
+                if (listener) {
+                    const mapping = this.#awareness.getStates();
+                    const editing_mapping: Record<string, string[]> = {};
+                    for (const {
+                        userId,
+                        editing,
+                        updated,
+                    } of mapping.values()) {
+                        // Only return the status with refresh time within 10 seconds
+                        if (
+                            userId &&
+                            editing &&
+                            updated &&
+                            typeof updated === 'number' &&
+                            updated + 1000 * 10 > Date.now()
+                        ) {
+                            if (!editing_mapping[editing])
+                                editing_mapping[editing] = [];
+                            editing_mapping[editing].push(userId);
+                        }
+                    }
+                    listener(
+                        new Map(
+                            Object.entries(editing_mapping).map(([k, v]) => [
+                                k,
+                                new Set(v),
+                            ])
+                        )
+                    );
+                }
+            },
+            200,
+            { maxWait: 1000 }
+        );
+
+        this.#awareness.setLocalStateField('userId', providers.userId);
+
+        this.#awareness.on('update', debounced_editing_notifier);
+
+        this.#blocks.observeDeep(events => {
+            const now = Date.now();
+
+            const keys = events.flatMap(e => {
+                if ((e.path?.length | 0) > 0) {
+                    return [
+                        [e.path[0], 'update'] as [string, ChangedStateKeys],
+                    ];
+                } else {
+                    return Array.from(e.changes.keys.entries()).map(
+                        ([k, { action }]) =>
+                            [k, action] as [string, ChangedStateKeys]
+                    );
+                }
+            });
+
+            EmitEvents(keys, this.#listener.get('updated'));
+
+            transact(this.#doc, () => {
+                for (const [key, action] of keys) {
+                    if (action === 'delete') {
+                        this.#block_updated.delete(key);
+                    } else {
+                        this.#block_updated.set(key, now);
+                    }
+                }
+            });
+        });
+    }
+
+    getUserId(): string {
+        return this.#provider.userId;
+    }
+
+    inspector() {
+        const resolve_block = (blocks: Record<string, any>, id: string) => {
+            const block = blocks[id];
+            if (block) {
+                return {
+                    ...block,
+                    children: block.children.map((id: string) =>
+                        resolve_block(blocks, id)
+                    ),
+                };
+            }
+        };
+
+        return {
+            save: () => {
+                const binary = encodeStateAsUpdate(this.#doc);
+                saveAs(
+                    new Blob([binary]),
+                    `affine_workspace_${new Date().toDateString()}.apk`
+                );
+            },
+            load: async () => {
+                const handles = await window.showOpenFilePicker({
+                    types: [
+                        {
+                            description: 'Affine Package',
+                            accept: {
+                                // eslint-disable-next-line @typescript-eslint/naming-convention
+                                'application/affine': ['.apk'],
+                            },
+                        },
+                    ],
+                });
+                const [file] = (await fromEvent(handles)) as File[];
+                const binary = await file.arrayBuffer();
+                await this.#provider.idb.clearData();
+                const doc = new Doc({ autoLoad: true, shouldLoad: true });
+                let updated = 0;
+                let isUpdated = false;
+                doc.on('update', () => {
+                    isUpdated = true;
+                    updated += 1;
+                });
+                setInterval(() => {
+                    if (updated > 0) updated -= 1;
+                }, 500);
+
+                const update_check = new Promise<void>(resolve => {
+                    const check = async () => {
+                        while (!isUpdated || updated > 0) {
+                            await sleep();
+                        }
+                        resolve();
+                    };
+                    check();
+                });
+                await new IndexeddbPersistence(this.#provider.idb.name, doc)
+                    .whenSynced;
+                applyUpdate(doc, new Uint8Array(binary));
+                await update_check;
+                console.log('load success');
+            },
+            parse: () => this.#doc.toJSON(),
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            parse_page: (page_id: string) => {
+                const blocks = this.#blocks.toJSON();
+                return resolve_block(blocks, page_id);
+            },
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            parse_pages: (resolve = false) => {
+                const blocks = this.#blocks.toJSON();
+                return Object.fromEntries(
+                    Object.entries(blocks)
+                        .filter(([, block]) => block.flavor === 'page')
+                        .map(([key, block]) => {
+                            if (resolve) {
+                                return resolve_block(blocks, key);
+                            } else {
+                                return [key, block];
+                            }
+                        })
+                );
+            },
+            clear: () => {
+                this.#blocks.clear();
+                this.#block_updated.clear();
+                this.#gatekeeper.clear();
+            },
+        };
+    }
+
+    async createBlock(
+        options: Pick<BlockItem<YjsContentOperation>, 'type' | 'flavor'> & {
+            uuid?: string;
+            binary?: ArrayBufferLike;
+        }
+    ): Promise<YjsBlockInstance> {
+        const uuid = options.uuid || `affine${nanoid(16)}`;
+        if (options.type === BlockTypes.binary) {
+            if (options.binary && options.binary instanceof ArrayBuffer) {
+                const array = new YArray();
+                array.insert(0, [options.binary]);
+                const block = {
+                    type: options.type,
+                    flavor: options.flavor,
+                    children: [] as string[],
+                    created: Date.now(),
+                    content: new YjsContentOperation(array),
+                    hash: sha3(Buffer.from(options.binary)),
+                };
+                await this.set_block(uuid, block);
+                return (await this.getBlock(uuid))!;
+            } else {
+                throw new Error(`Invalid binary type: ${options.binary}`);
+            }
+        } else {
+            const block = {
+                type: options.type,
+                flavor: options.flavor,
+                children: [] as string[],
+                created: Date.now(),
+                content: new YjsContentOperation(new YMap()),
+            };
+            await this.set_block(uuid, block);
+            return (await this.getBlock(uuid))!;
+        }
+    }
+
+    private get_updated(id: string) {
+        return this.#block_updated.get(id);
+    }
+
+    private get_creator(id: string) {
+        return this.#gatekeeper.getCreator(id);
+    }
+
+    private get_block_sync(id: string): YjsBlockInstance | undefined {
+        const cached = this.#block_caches.get(id);
+        if (cached) {
+            // Synchronous read cannot read binary
+            if (cached.type === BlockTypes.block) {
+                return cached;
+            }
+            return undefined;
+        }
+
+        const block = this.#blocks.get(id);
+
+        // Synchronous read cannot read binary
+        if (block && block.get('type') === BlockTypes.block) {
+            return new YjsBlockInstance({
+                id,
+                block,
+                setBlock: this.set_block.bind(this),
+                getUpdated: this.get_updated.bind(this),
+                getCreator: this.get_creator.bind(this),
+                getBlockInstance: this.get_block_sync.bind(this),
+            });
+        }
+
+        return undefined;
+    }
+
+    async getBlock(id: string): Promise<YjsBlockInstance | undefined> {
+        const block_instance = this.get_block_sync(id);
+        if (block_instance) return block_instance;
+        const block = this.#blocks.get(id);
+        if (block && block.get('type') === BlockTypes.binary) {
+            const binary = await this.#binaries.get(
+                block.get('hash') as string
+            );
+            if (binary) {
+                return new YjsBlockInstance({
+                    id,
+                    block,
+                    binary,
+                    setBlock: this.set_block.bind(this),
+                    getUpdated: this.get_updated.bind(this),
+                    getCreator: this.get_creator.bind(this),
+                    getBlockInstance: this.get_block_sync.bind(this),
+                });
+            }
+        }
+        return undefined;
+    }
+
+    async getBlockByFlavor(
+        flavor: BlockItem<YjsContentOperation>['flavor']
+    ): Promise<string[]> {
+        const keys: string[] = [];
+        this.#blocks.forEach((doc, key) => {
+            if (doc.get('flavor') === flavor) {
+                keys.push(key);
+            }
+        });
+
+        return keys;
+    }
+
+    async getBlockByType(
+        type: BlockItem<YjsContentOperation>['type']
+    ): Promise<string[]> {
+        const keys: string[] = [];
+        this.#blocks.forEach((doc, key) => {
+            if (doc.get('type') === type) {
+                keys.push(key);
+            }
+        });
+
+        return keys;
+    }
+
+    private async set_block(
+        key: string,
+        item: BlockItem<YjsContentOperation> & { hash?: string }
+    ): Promise<void> {
+        return new Promise<void>((resolve, reject) => {
+            const block = this.#blocks.get(key) || new YMap();
+            transact(this.#doc, () => {
+                // Insert only if the block doesn't exist yet
+                // Other modification operations are done in the block instance
+                let uploaded: Promise<void> | undefined;
+                if (!block.size) {
+                    const content = item.content[INTO_INNER]();
+                    if (!content) return reject();
+
+                    const children = new YArray();
+                    children.push(item.children);
+
+                    block.set('type', item.type);
+                    block.set('flavor', item.flavor);
+                    block.set('children', children);
+                    block.set('created', item.created);
+                    if (item.type === BlockTypes.block) {
+                        block.set('content', content);
+                    } else if (item.type === BlockTypes.binary && item.hash) {
+                        if (content instanceof YArray) {
+                            block.set('hash', item.hash);
+                            if (!this.#binaries.has(item.hash)) {
+                                uploaded = this.#binaries.set(
+                                    item.hash,
+                                    content
+                                );
+                            }
+                        } else {
+                            throw new Error(
+                                'binary content must be an buffer yarray'
+                            );
+                        }
+                    } else {
+                        throw new Error('invalid block type: ' + item.type);
+                    }
+
+                    this.#blocks.set(key, block);
+                }
+
+                if (item.flavor === 'page') {
+                    this.#awareness.setLocalStateField('editing', key);
+                    this.#awareness.setLocalStateField('updated', Date.now());
+                }
+                // References do not add delete restrictions
+                if (item.flavor === 'reference') {
+                    this.#gatekeeper.setCommon(key);
+                } else {
+                    this.#gatekeeper.setCreator(key);
+                }
+
+                if (uploaded) {
+                    // TODO: there should be a mechanism to retry the upload
+                    uploaded.catch(err => {
+                        // undo set on failure
+                        console.error('Failed to upload object: ', err);
+                        this.deleteBlocks([key]);
+                        reject(err);
+                    });
+                }
+                resolve();
+            });
+        });
+    }
+
+    async checkBlocks(keys: string[]): Promise<boolean> {
+        return (
+            keys.filter(key => !!this.#blocks.get(key)).length === keys.length
+        );
+    }
+
+    async deleteBlocks(keys: string[]): Promise<string[]> {
+        const [success, fail] = this.#gatekeeper.checkDeleteLists(keys);
+        transact(this.#doc, () => {
+            for (const key of success) {
+                this.#blocks.delete(key);
+            }
+        });
+        return fail;
+    }
+
+    on<S, R>(key: 'editing' | 'updated', listener: BlockListener<S, R>): void {
+        this.#listener.set(key, listener);
+    }
+
+    suspend(suspend: boolean) {
+        Suspend(suspend);
+    }
+
+    public history(): HistoryManager {
+        return this.#history;
+    }
+}

libs/datasource/jwt/src/adapter/yjs/listener.ts

@@ -0,0 +1,96 @@
+import { produce } from 'immer';
+import { debounce } from 'ts-debounce';
+import { YEvent } from 'yjs';
+import { BlockListener, ChangedStateKeys } from '../index';
+
+let listener_suspend = false;
+
+let listener_map = new Map<BlockListener, [string, ChangedStateKeys][]>();
+
+const debounced_suspend_notifier = debounce(
+    (listener?: BlockListener) => {
+        if (listener) {
+            listener_map = produce(listener_map, draft => {
+                const events = draft.get(listener);
+                if (events) {
+                    listener(new Map(events));
+                    draft.delete(listener);
+                }
+            });
+        }
+    },
+    500,
+    { maxWait: 2000 }
+);
+
+/**
+ * Suspend instant update event dispatch, extend to at least 500ms once, and up to 2000ms once when triggered continuously
+ * @param suspend true: suspend monitoring, false: resume monitoring
+ */
+export function Suspend(suspend: boolean) {
+    listener_suspend = produce(listener_suspend, draft => {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        draft = suspend;
+    });
+    if (!suspend && listener_map.size) {
+        listener_map = produce(listener_map, draft => {
+            for (const [listener, events] of draft) {
+                listener(new Map(events));
+            }
+            draft.clear();
+        });
+    }
+}
+
+export function EmitEvents(
+    events: [string, ChangedStateKeys][],
+    listener?: BlockListener
+) {
+    if (listener) {
+        if (listener_suspend) {
+            listener_map = produce(listener_map, draft => {
+                const old_events = listener_map.get(listener) || [];
+                draft.set(listener, [...old_events, ...events]);
+            });
+            debounced_suspend_notifier(listener);
+        } else {
+            listener(new Map(events));
+        }
+    }
+}
+
+export function ChildrenListenerHandler(
+    listeners: Map<string, BlockListener>,
+    event: YEvent<any>
+) {
+    if (listeners.size) {
+        const keys = Array.from(event.keys.entries()).map(
+            ([key, { action }]) => [key, action] as [string, ChangedStateKeys]
+        );
+        for (const listener of listeners.values()) {
+            EmitEvents(keys, listener);
+        }
+    }
+}
+
+export function ContentListenerHandler(
+    listeners: Map<string, BlockListener>,
+    events: YEvent<any>[]
+) {
+    if (listeners.size) {
+        const keys = events.flatMap(e => {
+            if ((e.path?.length | 0) > 0) {
+                return [[e.path[0], 'update'] as [string, 'update']];
+            } else {
+                return Array.from(e.changes.keys.entries()).map(
+                    ([k, { action }]) => [k, action] as [string, typeof action]
+                );
+            }
+        });
+        if (keys.length) {
+            for (const listener of listeners.values()) {
+                EmitEvents(keys, listener);
+            }
+        }
+    }
+}

libs/datasource/jwt/src/adapter/yjs/operation.ts

@@ -0,0 +1,388 @@
+/* eslint-disable max-lines */
+import {
+    AbstractType as YAbstractType,
+    Array as YArray,
+    Map as YMap,
+    Text as YText,
+} from 'yjs';
+
+import {
+    ArrayOperation,
+    BaseTypes,
+    BlockListener,
+    ContentOperation,
+    ContentTypes,
+    MapOperation,
+    Operable,
+    TextOperation,
+    TextToken,
+} from '../index';
+import { ChildrenListenerHandler, ContentListenerHandler } from './listener';
+
+const INTO_INNER = Symbol('INTO_INNER');
+
+export const DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_SYMBOL_INTO_INNER: typeof INTO_INNER =
+    INTO_INNER;
+
+function auto_get(root: ContentOperation, key: string): unknown | undefined {
+    const array = root.asArray();
+    if (array && !Number.isNaN(Number(key))) return array.get(Number(key));
+    const map = root.asMap();
+    if (map) return map.get(key);
+    const text = root.asText();
+    if (text) return text.toString();
+    console.error('auto_get unknown root', root, key);
+    return undefined;
+}
+
+function auto_set(root: ContentOperation, key: string, data: BaseTypes): void {
+    const array = root.asArray<BaseTypes>();
+    if (array && !Number.isNaN(Number(key))) {
+        return array.insert(Number(key), [data]);
+    }
+    const map = root.asMap<BaseTypes>();
+    if (map) {
+        return map.set(key, data);
+    }
+    const text = root.asText();
+    if (text && !Number.isNaN(Number(key)) && typeof data === 'string') {
+        return text.insert(Number(key), data);
+    }
+    console.error('autoSet unknown root or path', root, key, data);
+}
+
+export class YjsContentOperation implements ContentOperation {
+    readonly #content: YAbstractType<unknown>;
+
+    constructor(content: YAbstractType<any>) {
+        this.#content = content;
+    }
+
+    get length(): number {
+        if (this.#content instanceof YMap) {
+            return this.#content.size;
+        }
+        if (this.#content instanceof YArray || this.#content instanceof YText) {
+            return this.#content.length;
+        }
+        return 0;
+    }
+
+    createText(): YjsTextOperation {
+        return new YjsTextOperation(new YText());
+    }
+
+    createArray<
+        T extends ContentTypes = ContentOperation
+    >(): YjsArrayOperation<T> {
+        return new YjsArrayOperation(new YArray());
+    }
+
+    createMap<T extends ContentTypes = ContentOperation>(): YjsMapOperation<T> {
+        return new YjsMapOperation(new YMap());
+    }
+
+    asText(): YjsTextOperation | undefined {
+        if (this.#content instanceof YText) {
+            return new YjsTextOperation(this.#content);
+        }
+        return undefined;
+    }
+
+    asArray<T extends ContentTypes = ContentOperation>():
+        | YjsArrayOperation<T>
+        | undefined {
+        if (this.#content instanceof YArray) {
+            return new YjsArrayOperation(this.#content);
+        }
+        return undefined;
+    }
+
+    asMap<T extends ContentTypes = ContentOperation>():
+        | YjsMapOperation<T>
+        | undefined {
+        if (this.#content instanceof YMap) {
+            return new YjsMapOperation(this.#content);
+        }
+        return undefined;
+    }
+
+    autoGet(
+        root: ThisType<ContentOperation> | Record<string, unknown>,
+        path: string[]
+    ): unknown | undefined {
+        if (root) {
+            if (path.length === 0) {
+                return root;
+            } else if (root instanceof YjsContentOperation) {
+                const [key, ...rest] = path;
+                const new_root = auto_get(root, key);
+                if (new_root) {
+                    return this.autoGet(new_root as typeof root, rest);
+                }
+            } else if (typeof root === 'object') {
+                throw new Error(
+                    'autoGet must not get a non-value type, this is a deprecated behavior'
+                );
+            }
+        }
+        console.error('autoGet unknown root', root, path);
+        return undefined;
+    }
+
+    autoSet(
+        root: ThisType<ContentOperation>,
+        path: string[],
+        data: BaseTypes,
+        partial?: boolean
+    ): void {
+        if (root) {
+            if (path.length === 0) {
+                if (data && typeof data === 'object') {
+                    throw new Error(
+                        'autoSet must not set a non-value type, this is a deprecated behavior'
+                    );
+                } else {
+                    console.error('autoSet unknown data', root, path, data);
+                }
+                return;
+            }
+            if (root instanceof YjsContentOperation) {
+                if (path.length === 1) {
+                    const [key] = path;
+                    if (key) return auto_set(root, key, data);
+                    console.error('autoSet unknown path', root, path, data);
+                    return;
+                }
+                const [key, ...rest] = path;
+                const new_root = auto_get(root, key);
+                if (new_root && new_root instanceof YjsContentOperation) {
+                    return this.autoSet(new_root, rest, data, partial);
+                } else {
+                    throw new Error(
+                        'autoSet must not set a non-value type, this is a deprecated behavior'
+                    );
+                }
+            }
+        }
+        console.error('autoSet unknown root', root, path);
+    }
+
+    protected into_inner<T>(content: Operable<T>): T {
+        if (content instanceof YjsContentOperation) {
+            return content[INTO_INNER]() as unknown as T;
+        } else {
+            return content as T;
+        }
+    }
+
+    protected to_operable<T>(content: T): Operable<T> {
+        if (content instanceof YAbstractType) {
+            return new YjsContentOperation(content) as unknown as Operable<T>;
+        }
+        return content as Operable<T>;
+    }
+
+    [INTO_INNER](): YAbstractType<unknown> | undefined {
+        if (this.#content instanceof YAbstractType) {
+            return this.#content;
+        }
+        return undefined;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    private toJSON() {
+        return this.#content.toJSON();
+    }
+}
+
+class YjsTextOperation extends YjsContentOperation implements TextOperation {
+    readonly #content: YText;
+
+    constructor(content: YText) {
+        super(content);
+        this.#content = content;
+    }
+
+    insert(
+        index: number,
+        content: string,
+        format?: Record<string, string>
+    ): void {
+        this.#content.insert(index, content, format);
+    }
+
+    format(
+        index: number,
+        length: number,
+        format: Record<string, string>
+    ): void {
+        this.#content.format(index, length, format);
+    }
+
+    delete(index: number, length: number): void {
+        this.#content.delete(index, length);
+    }
+
+    setAttribute(name: string, value: BaseTypes) {
+        this.#content.setAttribute(name, value);
+    }
+
+    getAttribute<T extends BaseTypes = string>(name: string): T | undefined {
+        return this.#content.getAttribute(name);
+    }
+
+    override toString(): TextToken[] {
+        return this.#content.toDelta();
+    }
+}
+
+class YjsArrayOperation<T extends ContentTypes>
+    extends YjsContentOperation
+    implements ArrayOperation<T>
+{
+    readonly #content: YArray<T>;
+    readonly #listeners: Map<string, BlockListener>;
+
+    constructor(content: YArray<T>) {
+        super(content);
+        this.#content = content;
+        this.#listeners = new Map();
+
+        this.#content.observe(event =>
+            ChildrenListenerHandler(this.#listeners, event)
+        );
+    }
+
+    on(name: string, listener: BlockListener) {
+        this.#listeners.set(name, listener);
+    }
+
+    off(name: string) {
+        this.#listeners.delete(name);
+    }
+
+    insert(index: number, content: Array<Operable<T>>): void {
+        this.#content.insert(
+            index,
+            content.map(v => this.into_inner(v))
+        );
+    }
+
+    delete(index: number, length: number): void {
+        this.#content.delete(index, length);
+    }
+
+    push(content: Array<Operable<T>>): void {
+        this.#content.push(content.map(v => this.into_inner(v)));
+    }
+
+    unshift(content: Array<Operable<T>>): void {
+        this.#content.unshift(content.map(v => this.into_inner(v)));
+    }
+
+    get(index: number): Operable<T> | undefined {
+        const content = this.#content.get(index);
+        if (content) return this.to_operable(content);
+        return undefined;
+    }
+
+    private get_internal(index: number): T {
+        return this.#content.get(index);
+    }
+
+    slice(start?: number, end?: number): Operable<T>[] {
+        return this.#content.slice(start, end).map(v => this.to_operable(v));
+    }
+
+    map<R = unknown>(callback: (value: T, index: number) => R): R[] {
+        return this.#content.map((value, index) => callback(value, index));
+    }
+
+    // Traverse, if callback returns false, stop traversing
+    forEach(callback: (value: T, index: number) => boolean) {
+        for (let i = 0; i < this.#content.length; i++) {
+            const ret = callback(this.get_internal(i), i);
+            if (ret === false) {
+                break;
+            }
+        }
+    }
+
+    find<R = unknown>(
+        callback: (value: T, index: number) => boolean
+    ): R | undefined {
+        let result: R | undefined = undefined;
+        this.forEach((value, i) => {
+            const found = callback(value, i);
+            if (found) {
+                result = value as unknown as R;
+                return false;
+            }
+            return true;
+        });
+        return result;
+    }
+
+    findIndex(callback: (value: T, index: number) => boolean): number {
+        let position = -1;
+        this.forEach((value, i) => {
+            const found = callback(value, i);
+            if (found) {
+                position = i;
+                return false;
+            }
+            return true;
+        });
+        return position;
+    }
+}
+
+class YjsMapOperation<T extends ContentTypes>
+    extends YjsContentOperation
+    implements MapOperation<T>
+{
+    readonly #content: YMap<T>;
+    readonly #listeners: Map<string, BlockListener>;
+
+    constructor(content: YMap<T>) {
+        super(content);
+        this.#content = content;
+        this.#listeners = new Map();
+
+        content?.observeDeep(events =>
+            ContentListenerHandler(this.#listeners, events)
+        );
+    }
+
+    on(name: string, listener: BlockListener) {
+        this.#listeners.set(name, listener);
+    }
+
+    off(name: string) {
+        this.#listeners.delete(name);
+    }
+
+    set(key: string, value: Operable<T>): void {
+        if (value instanceof YjsContentOperation) {
+            const content = value[INTO_INNER]();
+            if (content) this.#content.set(key, content as unknown as T);
+        } else {
+            this.#content.set(key, value as T);
+        }
+    }
+
+    get(key: string): Operable<T> | undefined {
+        const content = this.#content.get(key);
+        if (content) return this.to_operable(content);
+        return undefined;
+    }
+
+    delete(key: string): void {
+        this.#content.delete(key);
+    }
+
+    has(key: string): boolean {
+        return this.#content.has(key);
+    }
+}

libs/datasource/jwt/src/block/abstract.ts

@@ -0,0 +1,372 @@
+import {
+    BlockInstance,
+    BlockListener,
+    BlockPosition,
+    ContentOperation,
+    ContentTypes,
+    HistoryManager,
+    MapOperation,
+} from '../adapter';
+import {
+    BlockTypes,
+    BlockTypeKeys,
+    BlockFlavors,
+    BlockFlavorKeys,
+} from '../types';
+import { getLogger } from '../utils';
+
+declare const JWT_DEV: boolean;
+const logger = getLogger('BlockDB:block');
+const logger_debug = getLogger('debug:BlockDB:block');
+
+const GET_BLOCK = Symbol('GET_BLOCK');
+const SET_PARENT = Symbol('SET_PARENT');
+
+export class AbstractBlock<
+    B extends BlockInstance<C>,
+    C extends ContentOperation
+> {
+    readonly #id: string;
+    readonly #block: BlockInstance<C>;
+    readonly #history: HistoryManager;
+    readonly #root?: AbstractBlock<B, C>;
+    readonly #parent_listener: Map<string, BlockListener>;
+
+    #parent?: AbstractBlock<B, C>;
+
+    constructor(
+        block: B,
+        root?: AbstractBlock<B, C>,
+        parent?: AbstractBlock<B, C>
+    ) {
+        this.#id = block.id;
+        this.#block = block;
+        this.#history = this.#block.scopedHistory([this.#id]);
+
+        this.#root = root;
+        this.#parent_listener = new Map();
+        this.#parent = parent;
+        JWT_DEV && logger_debug(`init: exists ${this.#id}`);
+    }
+
+    public get root() {
+        return this.#root;
+    }
+
+    protected get parent_node() {
+        return this.#parent;
+    }
+
+    protected _getParentPage(warning = true): string | undefined {
+        if (this.flavor === 'page') {
+            return this.#block.id;
+        } else if (!this.#parent) {
+            if (warning && this.flavor !== 'workspace') {
+                console.warn('parent not found');
+            }
+            return undefined;
+        } else {
+            return this.#parent.parent_page;
+        }
+    }
+
+    public get parent_page(): string | undefined {
+        return this._getParentPage();
+    }
+
+    public on(
+        event: 'content' | 'children' | 'parent',
+        name: string,
+        callback: BlockListener
+    ) {
+        if (event === 'parent') {
+            this.#parent_listener.set(name, callback);
+        } else {
+            this.#block.on(event, name, callback);
+        }
+    }
+
+    public off(event: 'content' | 'children' | 'parent', name: string) {
+        if (event === 'parent') {
+            this.#parent_listener.delete(name);
+        } else {
+            this.#block.off(event, name);
+        }
+    }
+
+    public addChildrenListener(name: string, listener: BlockListener) {
+        this.#block.addChildrenListener(name, listener);
+    }
+
+    public removeChildrenListener(name: string) {
+        this.#block.removeChildrenListener(name);
+    }
+
+    public addContentListener(name: string, listener: BlockListener) {
+        this.#block.addContentListener(name, listener);
+    }
+
+    public removeContentListener(name: string) {
+        this.#block.removeContentListener(name);
+    }
+
+    public getContent<
+        T extends ContentTypes = ContentOperation
+    >(): MapOperation<T> {
+        if (this.#block.type === BlockTypes.block) {
+            return this.#block.content.asMap() as MapOperation<T>;
+        }
+        throw new Error(
+            `this block not a structured block: ${this.#id}, ${
+                this.#block.type
+            }`
+        );
+    }
+
+    public getBinary(): ArrayBuffer | undefined {
+        if (this.#block.type === BlockTypes.binary) {
+            return this.#block.content.asArray<ArrayBuffer>()?.get(0);
+        }
+        throw new Error('this block not a binary block');
+    }
+
+    public get<R = unknown>(path: string[]): R {
+        const content = this.getContent();
+        return content.autoGet(content, path) as R;
+    }
+
+    public set<V = unknown>(path: string[], value: V, partial?: boolean) {
+        const content = this.getContent();
+        content.autoSet(content, path, value, partial);
+    }
+
+    private get_date_text(timestamp?: number): string | undefined {
+        try {
+            if (timestamp && !Number.isNaN(timestamp)) {
+                return new Date(timestamp)
+                    .toISOString()
+                    .split('T')[0]
+                    .replace(/-/g, '');
+            }
+            // eslint-disable-next-line no-empty
+        } catch (e) {}
+        return undefined;
+    }
+
+    // Last update UTC time
+    public get lastUpdated(): number {
+        return this.#block.updated || this.#block.created;
+    }
+
+    private get last_updated_date(): string | undefined {
+        return this.get_date_text(this.lastUpdated);
+    }
+
+    // create UTC time
+    public get created(): number {
+        return this.#block.created;
+    }
+
+    private get created_date(): string | undefined {
+        return this.get_date_text(this.created);
+    }
+
+    // creator id
+    public get creator(): string | undefined {
+        return this.#block.creator;
+    }
+
+    [GET_BLOCK]() {
+        return this.#block;
+    }
+
+    [SET_PARENT](parent: AbstractBlock<B, C>) {
+        this.#parent = parent;
+        const states: Map<string, 'update'> = new Map([[parent.id, 'update']]);
+        for (const listener of this.#parent_listener.values()) {
+            listener(states);
+        }
+    }
+
+    /**
+     * Get document index tags
+     */
+    public getTags(): string[] {
+        const created = this.created_date;
+        const updated = this.last_updated_date;
+
+        return [
+            `id:${this.#id}`,
+            `type:${this.type}`,
+            `type:${this.flavor}`,
+            this.flavor === BlockFlavors.page && `type:doc`, // normal documentation
+            this.flavor === BlockFlavors.tag && `type:card`, // tag document
+            // this.type === ??? && `type:theorem`, // global marked math formula
+            created && `created:${created}`,
+            updated && `updated:${updated}`,
+        ].filter((v): v is string => !!v);
+    }
+
+    /**
+     * current document instance id
+     */
+    public get id(): string {
+        return this.#id;
+    }
+
+    /**
+     * current block type
+     */
+    public get type(): typeof BlockTypes[BlockTypeKeys] {
+        return this.#block.type;
+    }
+
+    /**
+     * current block flavor
+     */
+    public get flavor(): typeof BlockFlavors[BlockFlavorKeys] {
+        return this.#block.flavor;
+    }
+
+    // TODO: flavor needs optimization
+    setFlavor(flavor: typeof BlockFlavors[BlockFlavorKeys]) {
+        this.#block.setFlavor(flavor);
+    }
+
+    public get children(): string[] {
+        return this.#block.children;
+    }
+
+    /**
+     * Insert sub-Block
+     * @param block Block instance
+     * @param position Insertion position, if it is empty, it will be inserted at the end. If the block already exists, the position will be moved
+     * @returns
+     */
+    public async insertChildren(
+        block: AbstractBlock<B, C>,
+        position?: BlockPosition
+    ) {
+        JWT_DEV && logger(`insertChildren: start`);
+
+        if (block.id === this.#id) return; // avoid self-reference
+        if (
+            this.type !== BlockTypes.block || // binary cannot insert subblocks
+            (block.type !== BlockTypes.block &&
+                this.flavor !== BlockFlavors.workspace) // binary can only be inserted into workspace
+        ) {
+            throw new Error('insertChildren: binary not allow insert children');
+        }
+
+        this.#block.insertChildren(block[GET_BLOCK](), position);
+        block[SET_PARENT](this);
+    }
+
+    public hasChildren(id: string): boolean {
+        return this.#block.hasChildren(id);
+    }
+
+    /**
+     * Get an instance of the child Block
+     * @param blockId block id
+     * @returns
+     */
+    protected get_children(blockId?: string): BlockInstance<C>[] {
+        JWT_DEV && logger(`get children: ${blockId}`);
+        return this.#block.getChildren([blockId]);
+    }
+
+    public removeChildren(blockId?: string) {
+        this.#block.removeChildren([blockId]);
+    }
+
+    public remove() {
+        JWT_DEV && logger(`remove: ${this.id}`);
+        if (this.flavor !== BlockFlavors.workspace) {
+            // Pages other than workspace have parents
+            this.parent_node!.removeChildren(this.id);
+        }
+    }
+
+    public update(path: string[], value: Record<string, any>) {
+        this.set(path, value);
+    }
+
+    private insert_blocks(
+        parentNode: AbstractBlock<B, C> | undefined,
+        blocks: AbstractBlock<B, C>[],
+        placement: 'before' | 'after',
+        referenceNode?: AbstractBlock<B, C>
+    ) {
+        if (!blocks || blocks.length === 0 || !parentNode) {
+            return;
+        }
+        // TODO: array equal
+        if (
+            !referenceNode &&
+            parentNode.children.join('') ===
+                blocks.map(node => node.id).join('')
+        ) {
+            return;
+        }
+        blocks.forEach(block => {
+            if (block.parent_node) {
+                block.remove();
+            }
+
+            const placement_info = {
+                [placement]:
+                    referenceNode?.id ||
+                    (parentNode.hasChildNodes() &&
+                        parentNode.children[
+                            placement === 'before'
+                                ? 0
+                                : parentNode.children.length - 1
+                        ]),
+            };
+
+            parentNode.insertChildren(
+                block,
+                placement_info[placement] ? placement_info : undefined
+            );
+        });
+    }
+
+    prepend(...blocks: AbstractBlock<B, C>[]) {
+        this.insert_blocks(this, blocks.reverse(), 'before');
+    }
+
+    append(...blocks: AbstractBlock<B, C>[]) {
+        this.insert_blocks(this, blocks, 'after');
+    }
+
+    before(...blocks: AbstractBlock<B, C>[]) {
+        this.insert_blocks(this.parent_node, blocks, 'before', this);
+    }
+
+    after(...blocks: AbstractBlock<B, C>[]) {
+        this.insert_blocks(this.parent_node, blocks.reverse(), 'after', this);
+    }
+
+    hasChildNodes() {
+        return this.children.length > 0;
+    }
+
+    hasParent(blockId?: string) {
+        let parent = this.parent_node;
+        while (parent) {
+            if (parent.id === blockId) {
+                return true;
+            }
+            parent = parent.parent_node;
+        }
+        return false;
+    }
+
+    /**
+     * TODO: scoped history
+     */
+    public get history(): HistoryManager {
+        return this.#history;
+    }
+}

libs/datasource/jwt/src/block/base.ts

@@ -0,0 +1,321 @@
+import {
+    ArrayOperation,
+    BlockInstance,
+    ContentOperation,
+    ContentTypes,
+    InternalPlainObject,
+    MapOperation,
+} from '../adapter';
+import { BlockItem } from '../types';
+import { getLogger } from '../utils';
+
+import { AbstractBlock } from './abstract';
+import { BlockCapability } from './capability';
+
+const logger = getLogger('BlockDB:block');
+
+// TODO
+export interface Decoration extends InternalPlainObject {
+    key: string;
+    value: unknown;
+}
+
+type Validator = <T>(value: T | undefined) => boolean | void;
+
+export type IndexMetadata = Readonly<{
+    content?: string;
+    reference?: string;
+    tags: string[];
+}>;
+export type QueryMetadata = Readonly<
+    {
+        [key: string]: number | string | string[] | undefined;
+    } & Omit<BlockItem<any>, 'content'>
+>;
+
+export type ReadableContentExporter<
+    R = string,
+    T extends ContentTypes = ContentOperation
+> = (content: MapOperation<T>) => R;
+type GetExporter<R> = (
+    block: BlockInstance<any>
+) => Readonly<[string, ReadableContentExporter<R, any>]>[];
+
+type Exporters = {
+    content: GetExporter<string>;
+    metadata: GetExporter<Array<[string, number | string | string[]]>>;
+    tag: GetExporter<string[]>;
+};
+
+export class BaseBlock<
+    B extends BlockInstance<C>,
+    C extends ContentOperation
+> extends AbstractBlock<B, C> {
+    readonly #exporters?: Exporters;
+    readonly #content_exporters_getter: () => Map<
+        string,
+        ReadableContentExporter<string, any>
+    >;
+    readonly #metadata_exporters_getter: () => Map<
+        string,
+        ReadableContentExporter<
+            Array<[string, number | string | string[]]>,
+            any
+        >
+    >;
+    readonly #tag_exporters_getter: () => Map<
+        string,
+        ReadableContentExporter<string[], any>
+    >;
+
+    #validators: Map<string, Validator> = new Map();
+
+    constructor(
+        block: B,
+        root?: AbstractBlock<B, C>,
+        parent?: AbstractBlock<B, C>,
+        exporters?: Exporters
+    ) {
+        super(block, root, parent);
+
+        this.#exporters = exporters;
+        this.#content_exporters_getter = () =>
+            new Map(exporters?.content(block));
+        this.#metadata_exporters_getter = () =>
+            new Map(exporters?.metadata(block));
+        this.#tag_exporters_getter = () => new Map(exporters?.tag(block));
+    }
+
+    get parent() {
+        return this.parent_node as BaseBlock<B, C> | undefined;
+    }
+
+    private get decoration(): ArrayOperation<Decoration> | undefined {
+        const content = this.getContent<ArrayOperation<Decoration>>();
+        if (!content.has('decoration')) {
+            const decoration = content.createArray<Decoration>();
+            content.set('decoration', decoration);
+        }
+        return content.get('decoration')?.asArray();
+    }
+
+    getDecoration<T = unknown>(key: string): T | undefined {
+        const decoration = this.decoration?.find<Decoration>(
+            decoration => decoration.key === key
+        );
+        if (this.validate(key, decoration?.value)) {
+            return decoration?.value as T;
+        }
+        return undefined;
+    }
+
+    getDecorations<T = Record<string, unknown>>(): T {
+        const decorations = {} as T;
+        this.decoration?.forEach(decoration => {
+            const value = this.validate(decoration.key, decoration.value)
+                ? decoration.value
+                : undefined;
+            // @ts-ignore
+            decorations[decoration.key] = value;
+        });
+        return decorations;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    getCapability<T extends BlockCapability>(key: string): T | undefined {
+        // TODO: Capability api design
+        return undefined;
+    }
+
+    setDecoration(key: string, value: unknown) {
+        if (!this.validate(key, value)) {
+            throw new Error(`set [${key}] error: validate error.`);
+        }
+        const decoration = { key, value };
+        const index =
+            this.decoration?.findIndex(decoration => decoration.key === key) ??
+            -1;
+        if (index > -1) {
+            this.decoration?.delete(index, 1);
+            this.decoration?.insert(index, [decoration]);
+        } else {
+            this.decoration?.insert(this.decoration?.length, [decoration]);
+        }
+    }
+
+    removeDecoration(key: string) {
+        const index =
+            this.decoration?.findIndex(decoration => decoration.key === key) ??
+            -1;
+        if (index > -1) {
+            this.decoration?.delete(index, 1);
+        }
+    }
+
+    clearDecoration() {
+        this.decoration?.delete(0, this.decoration?.length);
+    }
+
+    setValidator(key: string, validator?: Validator) {
+        if (validator) {
+            this.#validators.set(key, validator);
+        } else {
+            this.#validators.delete(key);
+        }
+    }
+
+    private validate(key: string, value: unknown): boolean {
+        const validate = this.#validators.get(key);
+        if (validate) {
+            return validate(value) === false ? false : true;
+        }
+        return true;
+    }
+
+    get group(): BaseBlock<B, C> | undefined {
+        if (this.flavor === 'group') {
+            return this;
+        }
+        return this.parent?.group;
+    }
+
+    /**
+     * Get an instance of the child Block
+     * @param blockId block id
+     */
+    private get_children_instance(blockId?: string): BaseBlock<B, C>[] {
+        return this.get_children(blockId).map(
+            block => new BaseBlock(block, this.root, this, this.#exporters)
+        );
+    }
+
+    private get_indexable_metadata() {
+        const metadata: Record<string, number | string | string[]> = {};
+        for (const [name, exporter] of this.#metadata_exporters_getter()) {
+            try {
+                for (const [key, val] of exporter(this.getContent())) {
+                    metadata[key] = val;
+                }
+            } catch (err) {
+                logger(`Failed to export metadata: ${name}`, err);
+            }
+        }
+        try {
+            const parent_page = this._getParentPage(false);
+            if (parent_page) metadata['page'] = parent_page;
+            if (this.group) metadata['group'] = this.group.id;
+            if (this.parent) metadata['parent'] = this.parent.id;
+        } catch (e) {
+            logger(`Failed to export default metadata`, e);
+        }
+
+        return metadata;
+    }
+
+    public getQueryMetadata(): QueryMetadata {
+        return {
+            type: this.type,
+            flavor: this.flavor,
+            creator: this.creator,
+            children: this.children,
+            created: this.created,
+            updated: this.lastUpdated,
+            ...this.get_indexable_metadata(),
+        };
+    }
+
+    private get_indexable_content(): string | undefined {
+        const contents = [];
+        for (const [name, exporter] of this.#content_exporters_getter()) {
+            try {
+                const content = exporter(this.getContent());
+                if (content) contents.push(content);
+            } catch (err) {
+                logger(`Failed to export content: ${name}`, err);
+            }
+        }
+        if (!contents.length) {
+            try {
+                const content = this.getContent() as any;
+                return JSON.stringify(content['toJSON']());
+                // eslint-disable-next-line no-empty
+            } catch (e) {}
+        }
+        return contents.join('\n');
+    }
+
+    private get_indexable_tags(): string[] {
+        const tags: string[] = [];
+        for (const [name, exporter] of this.#tag_exporters_getter()) {
+            try {
+                tags.push(...exporter(this.getContent()));
+            } catch (err) {
+                logger(`Failed to export tags: ${name}`, err);
+            }
+        }
+
+        return tags;
+    }
+
+    public getIndexMetadata(): IndexMetadata {
+        return {
+            content: this.get_indexable_content(),
+            reference: '', // TODO: bibliography
+            tags: [...this.getTags(), ...this.get_indexable_tags()],
+        };
+    }
+
+    // ======================================
+    // DOM like apis
+    // ======================================
+
+    get firstChild() {
+        return this.get_children_instance(this.children[0]);
+    }
+
+    get lastChild() {
+        const children = this.children;
+        return this.get_children_instance(children[children.length - 1]);
+    }
+
+    get nextSibling(): BaseBlock<B, C> | undefined {
+        if (this.parent) {
+            const parent = this.parent;
+            const children = parent.children;
+            const index = children.indexOf(this.id);
+            return parent.get_children_instance(children[index + 1])[0];
+        }
+        return undefined;
+    }
+
+    get nextSiblings(): BaseBlock<B, C>[] {
+        if (this.parent) {
+            const parent = this.parent;
+            const children = parent.children;
+            const index = children.indexOf(this.id);
+            return (
+                children
+                    .slice(index + 1)
+                    .flatMap(id => parent.get_children_instance(id)) || []
+            );
+        }
+        return [];
+    }
+
+    get previousSibling(): BaseBlock<B, C> | undefined {
+        if (this.parent) {
+            const parent = this.parent;
+            const children = parent.children;
+            const index = children.indexOf(this.id);
+            return parent.get_children_instance(children[index - 1])[0];
+        }
+        return undefined;
+    }
+
+    contains(block: AbstractBlock<B, C>) {
+        if (this === block) {
+            return true;
+        }
+        return this.hasChildren(block.id);
+    }
+}

libs/datasource/jwt/src/block/capability.ts

@@ -0,0 +1,15 @@
+import { BlockSearchItem } from './index';
+
+export class BlockCapability {
+    // Accept a block instance, check its type, content data structure
+    // Does it meet the structural requirements of the current capability
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    protected check_block(block: BlockSearchItem): boolean {
+        return true;
+    }
+
+    // data structure upgrade
+    protected migration(): void {
+        // TODO: need to override
+    }
+}

libs/datasource/jwt/src/block/index.ts

@@ -0,0 +1,12 @@
+import type { BlockMetadata } from './indexer';
+
+export type BlockSearchItem = Partial<
+    BlockMetadata & {
+        readonly content: string;
+    }
+>;
+
+export { BaseBlock } from './base';
+export type { Decoration, ReadableContentExporter } from './base';
+export { BlockIndexer } from './indexer';
+export type { BlockCapability } from './capability';

libs/datasource/jwt/src/block/indexer.ts

@@ -0,0 +1,316 @@
+import { deflateSync, inflateSync, strToU8, strFromU8 } from 'fflate';
+import { Document as DocumentIndexer, DocumentSearchOptions } from 'flexsearch';
+import { get, set, keys, del, createStore } from 'idb-keyval';
+import produce from 'immer';
+import LRUCache from 'lru-cache';
+import sift, { Query } from 'sift';
+
+import {
+    AsyncDatabaseAdapter,
+    BlockInstance,
+    ChangedStates,
+    ContentOperation,
+} from '../adapter';
+import { BlockFlavors } from '../types';
+import { BlockEventBus, getLogger } from '../utils';
+
+import { BaseBlock, IndexMetadata, QueryMetadata } from './base';
+
+declare const JWT_DEV: boolean;
+
+const logger = getLogger('BlockDB:indexing');
+const logger_debug = getLogger('debug:BlockDB:indexing');
+
+type ChangedState = ChangedStates extends Map<unknown, infer R> ? R : never;
+
+export type BlockMetadata = QueryMetadata & { readonly id: string };
+
+function tokenizeZh(text: string) {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    const tokenizer = Intl?.['v8BreakIterator'];
+    if (tokenizer) {
+        const it = tokenizer(['zh-CN'], { type: 'word' });
+        it.adoptText(text);
+        const words = [];
+
+        let cur = 0,
+            prev = 0;
+
+        while (cur < text.length) {
+            prev = cur;
+            cur = it.next();
+            words.push(text.substring(prev, cur));
+        }
+
+        return words;
+    }
+    // eslint-disable-next-line no-control-regex
+    return text.replace(/[\x00-\x7F]/g, '').split('');
+}
+
+type IdbInstance = {
+    get: (key: string) => Promise<ArrayBufferLike | undefined>;
+    set: (key: string, value: ArrayBufferLike) => Promise<void>;
+    keys: () => Promise<string[]>;
+    delete: (key: string) => Promise<void>;
+};
+
+type BlockIdbInstance = {
+    index: IdbInstance;
+    metadata: IdbInstance;
+};
+
+function initIndexIdb(workspace: string): BlockIdbInstance {
+    const index = createStore(`${workspace}_index`, 'index');
+    const metadata = createStore(`${workspace}_metadata`, 'metadata');
+    return {
+        index: {
+            get: (key: string) => get<ArrayBufferLike>(key, index),
+            set: (key: string, value: ArrayBufferLike) =>
+                set(key, value, index),
+            keys: () => keys(index),
+            delete: (key: string) => del(key, index),
+        },
+        metadata: {
+            get: (key: string) => get<ArrayBufferLike>(key, metadata),
+            set: (key: string, value: ArrayBufferLike) =>
+                set(key, value, metadata),
+            keys: () => keys(metadata),
+            delete: (key: string) => del(key, metadata),
+        },
+    };
+}
+
+type BlockIndexedContent = {
+    index: IndexMetadata;
+    query: QueryMetadata;
+};
+
+export type QueryIndexMetadata = Query<QueryMetadata>;
+
+export class BlockIndexer<
+    A extends AsyncDatabaseAdapter<C>,
+    B extends BlockInstance<C>,
+    C extends ContentOperation
+> {
+    readonly #adapter: A;
+    readonly #idb: BlockIdbInstance;
+
+    readonly #block_indexer: DocumentIndexer<IndexMetadata>;
+    readonly #block_metadata: LRUCache<string, QueryMetadata>;
+    readonly #event_bus: BlockEventBus;
+
+    readonly #block_builder: (
+        block: BlockInstance<C>
+    ) => Promise<BaseBlock<B, C>>;
+
+    readonly #delay_index: { documents: Map<string, BaseBlock<B, C>> };
+
+    constructor(
+        adapter: A,
+        workspace: string,
+        block_builder: (block: BlockInstance<C>) => Promise<BaseBlock<B, C>>,
+        event_bus: BlockEventBus
+    ) {
+        this.#adapter = adapter;
+        this.#idb = initIndexIdb(workspace);
+
+        this.#block_indexer = new DocumentIndexer({
+            document: {
+                id: 'id',
+                index: ['content', 'reference'],
+                tag: 'tags',
+            },
+            encode: tokenizeZh,
+            tokenize: 'forward',
+            context: true,
+        });
+        this.#block_metadata = new LRUCache({
+            max: 10240,
+            ttl: 1000 * 60 * 30,
+        });
+
+        this.#block_builder = block_builder;
+        this.#event_bus = event_bus;
+
+        this.#delay_index = { documents: new Map() };
+
+        this.#event_bus
+            .topic('reindex')
+            .on('reindex', this.content_reindex.bind(this), {
+                debounce: { wait: 1000, maxWait: 1000 * 10 },
+            });
+
+        this.#event_bus
+            .topic('save_index')
+            .on('save_index', this.save_index.bind(this), {
+                debounce: { wait: 1000 * 10, maxWait: 1000 * 20 },
+            });
+    }
+
+    private async content_reindex() {
+        const paddings: Record<string, BlockIndexedContent> = {};
+
+        this.#delay_index.documents = produce(
+            this.#delay_index.documents,
+            draft => {
+                for (const [k, block] of draft) {
+                    paddings[k] = {
+                        index: block.getIndexMetadata(),
+                        query: block.getQueryMetadata(),
+                    };
+                    draft.delete(k);
+                }
+            }
+        );
+        for (const [key, { index, query }] of Object.entries(paddings)) {
+            if (index.content) {
+                await this.#block_indexer.addAsync(key, index);
+                this.#block_metadata.set(key, query);
+            }
+        }
+        this.#event_bus.topic('save_index').emit();
+    }
+
+    private async refresh_index(block: BaseBlock<B, C>) {
+        const filter: string[] = [
+            BlockFlavors.page,
+            BlockFlavors.title,
+            BlockFlavors.heading1,
+            BlockFlavors.heading2,
+            BlockFlavors.heading3,
+            BlockFlavors.text,
+            BlockFlavors.todo,
+            BlockFlavors.reference,
+        ];
+        if (filter.includes(block.flavor)) {
+            this.#delay_index.documents = produce(
+                this.#delay_index.documents,
+                draft => {
+                    draft.set(block.id, block);
+                }
+            );
+
+            this.#event_bus.topic('reindex').emit();
+            return true;
+        }
+        logger_debug(`skip index ${block.flavor}: ${block.id}`);
+        return false;
+    }
+
+    async refreshIndex(id: string, state: ChangedState) {
+        JWT_DEV && logger(`refreshArticleIndex: ${id}`);
+        if (state === 'delete') {
+            this.#delay_index.documents = produce(
+                this.#delay_index.documents,
+                draft => {
+                    this.#block_indexer.remove(id);
+                    this.#block_metadata.delete(id);
+                    draft.delete(id);
+                }
+            );
+            return;
+        }
+        const block = await this.#adapter.getBlock(id);
+        if (block?.id === id) {
+            if (await this.refresh_index(await this.#block_builder(block))) {
+                JWT_DEV &&
+                    logger(
+                        state
+                            ? `refresh index: ${id}, ${state}`
+                            : `indexing: ${id}`
+                    );
+            } else {
+                JWT_DEV && logger(`skip index: ${id}, ${block.flavor}`);
+            }
+        } else {
+            JWT_DEV && logger(`refreshArticleIndex: ${id} not exists`);
+        }
+    }
+
+    async loadIndex() {
+        for (const key of await this.#idb.index.keys()) {
+            const content = await this.#idb.index.get(key);
+            if (content) {
+                const decoded = strFromU8(inflateSync(new Uint8Array(content)));
+                try {
+                    await this.#block_indexer.import(key, decoded as any);
+                } catch (e) {
+                    console.error(`Failed to load index ${key}`, e);
+                }
+            }
+        }
+        for (const key of await this.#idb.metadata.keys()) {
+            const content = await this.#idb.metadata.get(key);
+            if (content) {
+                const decoded = strFromU8(inflateSync(new Uint8Array(content)));
+                try {
+                    await this.#block_indexer.import(key, JSON.parse(decoded));
+                } catch (e) {
+                    console.error(`Failed to load index ${key}`, e);
+                }
+            }
+        }
+        return Array.from(this.#block_metadata.keys());
+    }
+
+    private async save_index() {
+        const idb = this.#idb;
+        await idb.index
+            .keys()
+            .then(keys => Promise.all(keys.map(key => idb.index.delete(key))));
+        await this.#block_indexer.export((key, data) => {
+            return idb.index.set(
+                String(key),
+                deflateSync(strToU8(data as any))
+            );
+        });
+        const metadata = this.#block_metadata;
+        await idb.metadata
+            .keys()
+            .then(keys =>
+                Promise.all(
+                    keys
+                        .filter(key => !metadata.has(key))
+                        .map(key => idb.metadata.delete(key))
+                )
+            );
+        for (const [key, data] of metadata.entries()) {
+            await idb.metadata.set(
+                key,
+                deflateSync(strToU8(JSON.stringify(data)))
+            );
+        }
+    }
+
+    public async inspectIndex() {
+        const index: Record<string | number, any> = {};
+        await this.#block_indexer.export((key, data) => {
+            index[key] = data;
+        });
+    }
+
+    public search(
+        part_of_title_or_content:
+            | string
+            | Partial<DocumentSearchOptions<boolean>>
+    ) {
+        return this.#block_indexer.search(part_of_title_or_content as string);
+    }
+
+    public query(query: QueryIndexMetadata) {
+        const matches: string[] = [];
+        const filter = sift<QueryMetadata>(query);
+        this.#block_metadata.forEach((value, key) => {
+            if (filter(value)) matches.push(key);
+        });
+        return matches;
+    }
+
+    public getMetadata(ids: string[]): Array<BlockMetadata> {
+        return ids
+            .filter(id => this.#block_metadata.has(id))
+            .map(id => ({ ...this.#block_metadata.get(id)!, id }));
+    }
+}

libs/datasource/jwt/src/index.ts

@@ -0,0 +1,592 @@
+/* eslint-disable max-lines */
+import { DocumentSearchOptions } from 'flexsearch';
+import LRUCache from 'lru-cache';
+
+import {
+    AsyncDatabaseAdapter,
+    YjsAdapter,
+    YjsInitOptions,
+    YjsContentOperation,
+    ChangedStates,
+    BlockListener,
+    BlockInstance,
+    ContentOperation,
+    HistoryManager,
+    ContentTypes,
+} from './adapter';
+import { YjsBlockInstance } from './adapter/yjs';
+import {
+    BaseBlock,
+    BlockIndexer,
+    BlockSearchItem,
+    ReadableContentExporter,
+} from './block';
+import { QueryIndexMetadata } from './block/indexer';
+import {
+    BlockTypes,
+    BlockTypeKeys,
+    BlockFlavors,
+    BucketBackend,
+    UUID,
+    BlockFlavorKeys,
+    BlockItem,
+    ExcludeFunction,
+} from './types';
+import { BlockEventBus, genUUID, getLogger } from './utils';
+
+declare const JWT_DEV: boolean;
+
+const logger = getLogger('BlockDB:client');
+// const logger_debug = getLogger('debug:BlockDB:client');
+
+const namedUuid = Symbol('namedUUID');
+
+type BlockUuid<T extends string> = T extends UUID<T> ? T : never;
+type BlockUuidOrType<T extends string> = T extends
+    | BlockTypeKeys
+    | BlockFlavorKeys
+    ? T
+    : T extends string
+    ? BlockUuid<T>
+    : never;
+
+type BlockInstanceValue = ExcludeFunction<BlockInstance<any>>;
+export type BlockMatcher = Partial<BlockInstanceValue>;
+
+type BlockExporters<R> = Map<
+    string,
+    [BlockMatcher, ReadableContentExporter<R, any>]
+>;
+type BlockClientOptions = {
+    content?: BlockExporters<string>;
+    metadata?: BlockExporters<Array<[string, number | string | string[]]>>;
+    tagger?: BlockExporters<string[]>;
+};
+
+export class BlockClient<
+    A extends AsyncDatabaseAdapter<C>,
+    B extends BlockInstance<C>,
+    C extends ContentOperation
+> {
+    readonly #adapter: A;
+    readonly #workspace: string;
+
+    // Maximum cache Block 8192, ttl 30 minutes
+    readonly #block_caches: LRUCache<string, BaseBlock<B, C>>;
+    readonly #block_indexer: BlockIndexer<A, B, C>;
+
+    readonly #exporters: {
+        readonly content: BlockExporters<string>;
+        readonly metadata: BlockExporters<
+            Array<[string, number | string | string[]]>
+        >;
+        readonly tag: BlockExporters<string[]>;
+    };
+
+    readonly #event_bus: BlockEventBus;
+
+    readonly #parent_mapping: Map<string, string[]>;
+    readonly #page_mapping: Map<string, string>;
+
+    readonly #root: { node?: BaseBlock<B, C> };
+
+    private constructor(
+        adapter: A,
+        workspace: string,
+        options?: BlockClientOptions
+    ) {
+        this.#adapter = adapter;
+        this.#workspace = workspace;
+
+        this.#block_caches = new LRUCache({ max: 8192, ttl: 1000 * 60 * 30 });
+
+        this.#exporters = {
+            content: options?.content || new Map(),
+            metadata: options?.metadata || new Map(),
+            tag: options?.tagger || new Map(),
+        };
+
+        this.#event_bus = new BlockEventBus();
+
+        this.#block_indexer = new BlockIndexer(
+            this.#adapter,
+            this.#workspace,
+            this.block_builder.bind(this),
+            this.#event_bus.topic('indexer')
+        );
+
+        this.#parent_mapping = new Map();
+        this.#page_mapping = new Map();
+        this.#adapter.on('editing', (states: ChangedStates) =>
+            this.#event_bus.topic('editing').emit(states)
+        );
+        this.#adapter.on('updated', (states: ChangedStates) =>
+            this.#event_bus.topic('updated').emit(states)
+        );
+
+        this.#event_bus
+            .topic<string[]>('rebuild_index')
+            .on('rebuild_index', this.rebuild_index.bind(this), {
+                debounce: { wait: 1000, maxWait: 1000 },
+            });
+
+        this.#root = {};
+    }
+
+    public addBlockListener(tag: string, listener: BlockListener) {
+        const bus = this.#event_bus.topic<ChangedStates>('updated');
+        if (tag !== 'index' || !bus.has(tag)) bus.on(tag, listener);
+        else console.error(`block listener ${tag} is reserved`);
+    }
+
+    public removeBlockListener(tag: string) {
+        this.#event_bus.topic('updated').off(tag);
+    }
+
+    public addEditingListener(
+        tag: string,
+        listener: BlockListener<Set<string>>
+    ) {
+        const bus =
+            this.#event_bus.topic<ChangedStates<Set<string>>>('editing');
+        if (tag !== 'index' || !bus.has(tag)) bus.on(tag, listener);
+        else console.error(`editing listener ${tag} is reserved`);
+    }
+
+    public removeEditingListener(tag: string) {
+        this.#event_bus.topic('editing').off(tag);
+    }
+
+    private inspector() {
+        return {
+            ...this.#adapter.inspector(),
+            indexed: () => this.#block_indexer.inspectIndex(),
+        };
+    }
+
+    private async rebuild_index(exists_ids?: string[]) {
+        JWT_DEV && logger(`rebuild index`);
+        const blocks = await this.#adapter.getBlockByType(BlockTypes.block);
+        const excluded = exists_ids || [];
+        await Promise.all(
+            blocks
+                .filter(id => !excluded.includes(id))
+                .map(id => this.#block_indexer.refreshIndex(id, 'add'))
+        );
+    }
+
+    public async buildIndex() {
+        JWT_DEV && logger(`buildIndex: start`);
+        // Skip the block index that exists in the metadata, assuming that the index of the block existing in the metadata is the latest, and modify this part if there is a problem
+        // Although there may be cases where the index is refreshed but the metadata is not refreshed, re-indexing will be automatically triggered after the block is changed
+        const exists_ids = await this.#block_indexer.loadIndex();
+        await this.rebuild_index(exists_ids);
+        this.addBlockListener('index', async states => {
+            await Promise.allSettled(
+                Array.from(states.entries()).map(([id, state]) => {
+                    if (state === 'delete') this.#block_caches.delete(id);
+                    return this.#block_indexer.refreshIndex(id, state);
+                })
+            );
+        });
+    }
+
+    /**
+     * Get a specific type of block, currently only the article type is supported
+     * @param block_type block type
+     * @returns
+     */
+    public async getByType(
+        block_type: BlockTypeKeys | BlockFlavorKeys
+    ): Promise<Map<string, BaseBlock<B, C>>> {
+        JWT_DEV && logger(`getByType: ${block_type}`);
+        const ids = [
+            ...this.#block_indexer.query({
+                type: BlockTypes[block_type as BlockTypeKeys],
+            }),
+            ...this.#block_indexer.query({
+                flavor: BlockFlavors[block_type as BlockFlavorKeys],
+            }),
+        ];
+        const docs = await Promise.all(
+            ids.map(id =>
+                this.get(id as BlockUuidOrType<typeof id>).then(
+                    doc => [id, doc] as const
+                )
+            )
+        );
+        return new Map(docs.filter(([, doc]) => doc.children.length));
+    }
+
+    /**
+     * research all
+     * @param part_of_title_or_content Title or content keyword, support Chinese
+     * @param part_of_title_or_content.index search range, optional values: title, ttl, content, reference
+     * @param part_of_title_or_content.tag tag, string or array of strings, supports multiple tags
+     * @param part_of_title_or_content.query keyword, support Chinese
+     * @param part_of_title_or_content.limit The limit of the number of search results, the default is 100
+     * @param part_of_title_or_content.offset search result offset, used for page turning, default is 0
+     * @param part_of_title_or_content.suggest Fuzzy matching, after enabling the content including some keywords can also be searched, the default is false
+     * @returns array of search results, each array is a list of attributed block ids
+     */
+    public search(
+        part_of_title_or_content:
+            | string
+            | Partial<DocumentSearchOptions<boolean>>
+    ) {
+        return this.#block_indexer.search(part_of_title_or_content);
+    }
+
+    /**
+     * Full text search, the returned results are grouped by page dimension
+     * @param part_of_title_or_content Title or content keyword, support Chinese
+     * @param part_of_title_or_content.index search range, optional values: title, ttl, content, reference
+     * @param part_of_title_or_content.tag tag, string or array of strings, supports multiple tags
+     * @param part_of_title_or_content.query keyword, support Chinese
+     * @param part_of_title_or_content.limit The limit of the number of search results, the default is 100
+     * @param part_of_title_or_content.offset search result offset, used for page turning, default is 0
+     * @param part_of_title_or_content.suggest Fuzzy matching, after enabling the content including some keywords can also be searched, the default is false
+     * @returns array of search results, each array is a page
+     */
+    public async searchPages(
+        part_of_title_or_content:
+            | string
+            | Partial<DocumentSearchOptions<boolean>>
+    ): Promise<BlockSearchItem[]> {
+        const promised_pages = await Promise.all(
+            this.search(part_of_title_or_content).flatMap(({ result }) =>
+                result.map(async id => {
+                    const page = this.#page_mapping.get(id as string);
+                    if (page) return page;
+                    const block = await this.get(id as BlockTypeKeys);
+                    return this.set_page(block);
+                })
+            )
+        );
+        const pages = [
+            ...new Set(promised_pages.filter((v): v is string => !!v)),
+        ];
+        return Promise.all(
+            this.#block_indexer.getMetadata(pages).map(async page => ({
+                content: this.get_decoded_content(
+                    await this.#adapter.getBlock(page.id)
+                ),
+                ...page,
+            }))
+        );
+    }
+
+    /**
+     * Inquire
+     * @returns array of search results
+     */
+    public query(query: QueryIndexMetadata): string[] {
+        return this.#block_indexer.query(query);
+    }
+
+    /**
+     * Get a fixed name, which has the same UUID in each workspace, and is automatically created when it does not exist
+     * Generally used to store workspace-level global configuration
+     * @param name block name
+     * @returns block instance
+     */
+    private async get_named_block(
+        name: string,
+        options?: { workspace?: boolean }
+    ): Promise<BaseBlock<B, C>> {
+        const block = await this.get(genUUID(name), {
+            flavor: options?.workspace
+                ? BlockFlavors.workspace
+                : BlockFlavors.page,
+            [namedUuid]: true,
+        });
+        return block;
+    }
+
+    /**
+     * Get the workspace block of the current instance
+     * @returns block instance
+     */
+    public async getWorkspace() {
+        if (!this.#root.node) {
+            this.#root.node = await this.get_named_block(this.#workspace, {
+                workspace: true,
+            });
+        }
+        return this.#root.node;
+    }
+
+    /**
+     * @deprecated custom data including access to ws configuration is unified through baseBlock.get/setDecoration(key).
+     * - Get the config of the workspace block of the current instance
+     * @returns MapOperation
+     */
+    public async getWorkspaceConfig<T = unknown>() {
+        return (await this.getWorkspace()).getContent<T>();
+    }
+
+    private async get_parent(id: string) {
+        const parents = this.#parent_mapping.get(id);
+        if (parents) {
+            const parent_block_id = parents[0];
+            if (!this.#block_caches.has(parent_block_id)) {
+                this.#block_caches.set(
+                    parent_block_id,
+                    await this.get(parent_block_id as BlockTypeKeys)
+                );
+            }
+            return this.#block_caches.get(parent_block_id);
+        }
+        return undefined;
+    }
+
+    private set_parent(parent: string, child: string) {
+        const parents = this.#parent_mapping.get(child);
+        if (parents?.length) {
+            if (!parents.includes(parent)) {
+                console.error('parent already exists', child, parents);
+                this.#parent_mapping.set(child, [...parents, parent]);
+            }
+        } else {
+            this.#parent_mapping.set(child, [parent]);
+        }
+    }
+
+    private set_page(block: BaseBlock<B, C>) {
+        const page = this.#page_mapping.get(block.id);
+        if (page) return page;
+        const parent_page = block.parent_page;
+        if (parent_page) {
+            this.#page_mapping.set(block.id, parent_page);
+            return parent_page;
+        }
+        return undefined;
+    }
+
+    registerContentExporter<T extends ContentTypes>(
+        name: string,
+        matcher: BlockMatcher,
+        exporter: ReadableContentExporter<string, T>
+    ) {
+        this.#exporters.content.set(name, [matcher, exporter]);
+        this.#event_bus.topic('rebuild_index').emit(); // // rebuild the index every time the content exporter is registered
+    }
+
+    unregisterContentExporter(name: string) {
+        this.#exporters.content.delete(name);
+        this.#event_bus.topic('rebuild_index').emit(); // Rebuild indexes every time content exporter logs out
+    }
+
+    registerMetadataExporter<T extends ContentTypes>(
+        name: string,
+        matcher: BlockMatcher,
+        exporter: ReadableContentExporter<
+            Array<[string, number | string | string[]]>,
+            T
+        >
+    ) {
+        this.#exporters.metadata.set(name, [matcher, exporter]);
+        this.#event_bus.topic('rebuild_index').emit(); // // rebuild the index every time the content exporter is registered
+    }
+
+    unregisterMetadataExporter(name: string) {
+        this.#exporters.metadata.delete(name);
+        this.#event_bus.topic('rebuild_index').emit(); // Rebuild indexes every time content exporter logs out
+    }
+
+    registerTagExporter<T extends ContentTypes>(
+        name: string,
+        matcher: BlockMatcher,
+        exporter: ReadableContentExporter<string[], T>
+    ) {
+        this.#exporters.tag.set(name, [matcher, exporter]);
+        this.#event_bus.topic('rebuild_index').emit(); // Reindex every tag exporter registration
+    }
+
+    unregisterTagExporter(name: string) {
+        this.#exporters.tag.delete(name);
+        this.#event_bus.topic('rebuild_index').emit(); // Reindex every time tag exporter logs out
+    }
+
+    private get_exporters<R>(
+        exporter_map: BlockExporters<R>,
+        block: BlockInstance<C>
+    ): Readonly<[string, ReadableContentExporter<R, any>]>[] {
+        const exporters = [];
+        for (const [name, [cond, exporter]] of exporter_map) {
+            const conditions = Object.entries(cond);
+            let matched = 0;
+            for (const [key, value] of conditions) {
+                if (block[key as keyof BlockInstanceValue] === value) {
+                    matched += 1;
+                }
+            }
+            if (matched === conditions.length)
+                exporters.push([name, exporter] as const);
+        }
+
+        return exporters;
+    }
+
+    private get_decoded_content(block?: BlockInstance<C>) {
+        if (block) {
+            const [exporter] = this.get_exporters(
+                this.#exporters.content,
+                block
+            );
+            if (exporter) {
+                const op = block.content.asMap();
+                if (op) return exporter[1](op);
+            }
+        }
+        return undefined;
+    }
+
+    private async block_builder(
+        block: BlockInstance<C>,
+        root?: BaseBlock<B, C>
+    ) {
+        return new BaseBlock(
+            block,
+            root,
+            (await this.get_parent(block.id)) || root,
+            {
+                content: block =>
+                    this.get_exporters(this.#exporters.content, block),
+                metadata: block =>
+                    this.get_exporters(this.#exporters.metadata, block),
+                tag: block => this.get_exporters(this.#exporters.tag, block),
+            }
+        );
+    }
+
+    /**
+     * Get a Block, which is automatically created if it does not exist
+     * @param block_id_or_type block id, create a new text block when BlockTypes/BlockFlavors are not provided, does not exist or is provided. If BlockTypes/BlockFlavors are provided, create a block of the corresponding type
+     * @param options.type The type of block created when block does not exist, the default is block
+     * @param options.flavor The flavor of the block created when the block does not exist, the default is text
+     * @param options.binary content of binary block, must be provided when type or block_id_or_type is binary
+     * @returns block instance
+     */
+    public async get<S extends string>(
+        block_id_or_type?: BlockUuidOrType<S>,
+        options?: {
+            type?: BlockItem<C>['type'];
+            flavor: BlockItem<C>['flavor'];
+            binary?: ArrayBuffer;
+            [namedUuid]?: boolean;
+        }
+    ): Promise<BaseBlock<B, C>> {
+        JWT_DEV && logger(`get: ${block_id_or_type}`);
+        const {
+            type = BlockTypes.block,
+            flavor = BlockFlavors.text,
+            binary,
+            [namedUuid]: is_named_uuid,
+        } = options || {};
+        if (block_id_or_type && this.#block_caches.has(block_id_or_type)) {
+            return this.#block_caches.get(block_id_or_type) as BaseBlock<B, C>;
+        } else {
+            const block =
+                (block_id_or_type &&
+                    (await this.#adapter.getBlock(block_id_or_type))) ||
+                (await this.#adapter.createBlock({
+                    uuid: is_named_uuid ? block_id_or_type : undefined,
+                    binary,
+                    type:
+                        block_id_or_type &&
+                        BlockTypes[block_id_or_type as BlockTypeKeys]
+                            ? BlockTypes[block_id_or_type as BlockTypeKeys]
+                            : type,
+                    flavor:
+                        block_id_or_type &&
+                        BlockFlavors[block_id_or_type as BlockFlavorKeys]
+                            ? BlockFlavors[block_id_or_type as BlockFlavorKeys]
+                            : flavor,
+                }));
+
+            const root = is_named_uuid ? undefined : await this.getWorkspace();
+
+            for (const child of block.children) {
+                this.set_parent(block.id, child);
+            }
+
+            const abstract_block = await this.block_builder(block, root);
+            this.set_page(abstract_block);
+            abstract_block.on('parent', 'client_hook', state => {
+                const [parent] = state.keys();
+                this.set_parent(parent, abstract_block.id);
+                this.set_page(abstract_block);
+            });
+            this.#block_caches.set(abstract_block.id, abstract_block);
+            return abstract_block;
+        }
+    }
+
+    public async getBlockByFlavor(
+        flavor: BlockItem<C>['flavor']
+    ): Promise<string[]> {
+        return await this.#adapter.getBlockByFlavor(flavor);
+    }
+
+    public getUserId(): string {
+        return this.#adapter.getUserId();
+    }
+
+    public has(block_ids: string[]): Promise<boolean> {
+        return this.#adapter.checkBlocks(block_ids);
+    }
+
+    /**
+     * Suspend instant update event dispatch, extend to a maximum of 500ms once, and a maximum of 2000ms when triggered continuously
+     * @param suspend true: suspend monitoring, false: resume monitoring
+     */
+    suspend(suspend: boolean) {
+        this.#adapter.suspend(suspend);
+    }
+
+    public get history(): HistoryManager {
+        return this.#adapter.history();
+    }
+
+    public static async init(
+        workspace: string,
+        options: Partial<YjsInitOptions & BlockClientOptions> = {}
+    ): Promise<BlockClientInstance> {
+        const instance = await YjsAdapter.init(workspace, {
+            backend: BucketBackend.YjsWebSocketAffine,
+            ...options,
+        });
+        return new BlockClient(instance, workspace, options);
+    }
+}
+
+export type BlockImplInstance = BaseBlock<
+    YjsBlockInstance,
+    YjsContentOperation
+>;
+export type BlockClientInstance = BlockClient<
+    YjsAdapter,
+    YjsBlockInstance,
+    YjsContentOperation
+>;
+
+export type BlockInitOptions = NonNullable<
+    Parameters<typeof BlockClient.init>[1]
+>;
+
+export type {
+    TextOperation,
+    ArrayOperation,
+    MapOperation,
+    ChangedStates,
+} from './adapter';
+export type {
+    BlockSearchItem,
+    Decoration as BlockDecoration,
+    ReadableContentExporter as BlockContentExporter,
+} from './block';
+export type { BlockTypeKeys } from './types';
+export { BlockTypes, BucketBackend as BlockBackend } from './types';
+export { isBlock } from './utils';
+export type { QueryIndexMetadata };

libs/datasource/jwt/src/types/block.ts

@@ -0,0 +1,81 @@
+import { ContentOperation } from '../adapter';
+import { RefMetadata } from './metadata';
+import { UUID } from './uuid';
+// base type of block
+// y_block: tree structure expression with YAbstractType as root
+// y_binary: arbitrary binary data
+export const BlockTypes = {
+    block: 'y_block' as const, // data block
+    binary: 'y_binary' as const, // binary data
+};
+
+// block flavor
+// block has the same basic structure
+// But different flavors provide different parsing of their content
+// TODO, how do blockdb BlockFlavors synchronize with the protocol of '@toeverything/components/editor-blocks'?
+export const BlockFlavors = {
+    workspace: 'workspace' as const, // workspace
+    page: 'page' as const, // page
+    group: 'group' as const, // group
+    title: 'title' as const, // title
+    text: 'text' as const, // text
+    heading1: 'heading1' as const, // heading 1
+    heading2: 'heading2' as const, // heading 2
+    heading3: 'heading3' as const, // heading 3
+    code: 'code' as const, // code block
+    todo: 'todo' as const, // todo
+    numbered: 'numbered' as const, // numbered list
+    bullet: 'bullet' as const, // bullet list
+    comments: 'comments' as const, // comments
+    tag: 'tag' as const, // tag
+    reference: 'reference' as const, // reference
+    image: 'image' as const, // image
+    file: 'file' as const, //file
+    audio: 'audio' as const, // audio
+    video: 'video' as const, // video
+    shape: 'shape' as const, // artboard shape
+    quote: 'quote' as const, // quote
+    toc: 'toc' as const, //directory
+    database: 'database' as const, //Multidimensional table
+    whiteboard: 'whiteboard' as const, // whiteboard
+    template: 'template' as const, // template
+    discussion: 'discussion' as const, // comment header
+    comment: 'comment' as const, // comment details
+    activity: 'activity' as const, // dynamic message
+    divider: 'divider' as const, // divider line
+    groupDivider: 'groupDivider' as const, // group divider
+    youtube: 'youtube' as const, // Youtube
+    figma: 'figma' as const, // figma
+    embedLink: 'embedLink' as const, //embed link
+    toggle: 'toggle' as const, // collapse the list
+    callout: 'callout' as const, // reminder
+    grid: 'grid' as const, // grid layout
+    gridItem: 'gridItem' as const, // grid layout children
+};
+
+export type BlockTypeKeys = keyof typeof BlockTypes;
+export type BlockFlavorKeys = keyof typeof BlockFlavors;
+
+export type BlockItem<C extends ContentOperation> = {
+    readonly type: typeof BlockTypes[BlockTypeKeys];
+    flavor: typeof BlockFlavors[BlockFlavorKeys];
+    children: string[];
+    readonly created: number; // creation time, UTC timestamp
+    readonly updated?: number; // update time, UTC timestamp
+    readonly creator?: string; // creator id
+    content: C; // Essentially what is stored here is either Uint8Array (binary resource) or YDoc (structured resource)
+};
+
+export type BlockPage = {
+    title?: string;
+    description?: string;
+    // preview
+    preview?: UUID<'00000000-0000-0000-0000-000000000000'>;
+    // Whether it is a draft, the draft will not be indexed
+    draft?: string;
+    // Expiration time, documents that exceed this time will be deleted
+    ttl?: number;
+    // Metadata, currently only bibtex definitions
+    // When metadata exists, it will be treated as a reference and can be searched by tag reference
+    metadata?: RefMetadata;
+};

libs/datasource/jwt/src/types/index.ts

@@ -0,0 +1,28 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+export { BlockTypes, BlockFlavors } from './block';
+export type { BlockTypeKeys, BlockFlavorKeys, BlockItem } from './block';
+export type { UUID } from './uuid';
+export type { ExcludeFunction } from './utils';
+
+function getLocation() {
+    try {
+        const { protocol, host } = window.location;
+        return { protocol, host };
+    } catch (e) {
+        return { protocol: 'http:', host: 'localhost' };
+    }
+}
+
+function getCollaborationPoint() {
+    const { protocol, host } = getLocation();
+    const ws = protocol.startsWith('https') ? 'wss' : 'ws';
+    const isOnline = host.endsWith('affine.pro');
+    const site = isOnline ? host : 'localhost:4200';
+    return `${ws}://${site}/collaboration/`;
+}
+
+export const BucketBackend = {
+    IndexedDB: 'idb',
+    WebSQL: 'websql',
+    YjsWebSocketAffine: getCollaborationPoint(),
+};

libs/datasource/jwt/src/types/metadata.ts

@@ -0,0 +1,203 @@
+export type BibTexArticle = {
+    entry: 'article';
+    author: string;
+    title: string;
+    journal: string;
+    year: string;
+    volume?: string;
+    number?: string;
+    pages?: string;
+    month?: string;
+    note?: string;
+};
+
+export type BibTexBook = {
+    entry: 'book';
+    author: string;
+    editor?: string;
+    title: string;
+    publisher: string;
+    year: string;
+    volume?: string;
+    number?: string;
+    series?: string;
+    address?: string;
+    edition?: string;
+    month?: string;
+    note?: string;
+};
+
+export type BibTexBooklet = {
+    entry: 'booklet';
+    title: string;
+    author?: string;
+    howpublished?: string;
+    address?: string;
+    month?: string;
+    year?: string;
+    note?: string;
+};
+
+export type BibTexConference = {
+    entry: 'conference' | 'inproceedings';
+    author: string;
+    title: string;
+    booktitle: string;
+    year: string;
+    editor?: string;
+    volume?: string;
+    number?: string;
+    series?: string;
+    pages?: string;
+    address?: string;
+    month?: string;
+    organization?: string;
+    publisher?: string;
+    note?: string;
+};
+
+export type BibTexInBook = {
+    entry: 'inbook';
+    author: string;
+    editor: string;
+    title: string;
+    chapter: string;
+    pages?: string;
+    publisher: string;
+    year: string;
+    volume?: string;
+    number?: string;
+    series?: string;
+    type?: string;
+    address?: string;
+    edition?: string;
+    month?: string;
+    note?: string;
+};
+
+export type BibTexInCollection = {
+    entry: 'incollection';
+    author: string;
+    title: string;
+    booktitle: string;
+    publisher: string;
+    year: string;
+    editor?: string;
+    volume?: string;
+    number?: string;
+    series?: string;
+    type?: string;
+    chapter?: string;
+    pages?: string;
+    address?: string;
+    edition?: string;
+    month?: string;
+    note?: string;
+};
+
+export type BibTexManual = {
+    entry: 'manual';
+    title: string;
+    author?: string;
+    organization?: string;
+    address?: string;
+    edition?: string;
+    month?: string;
+    year?: string;
+    note?: string;
+};
+
+export type BibTexMastersThesis = {
+    entry: 'mastersthesis';
+    author: string;
+    title: string;
+    school: string;
+    year: string;
+    type?: string;
+    address?: string;
+    month?: string;
+    note?: string;
+};
+
+export type BibTexMisc = {
+    entry: 'misc';
+    author?: string;
+    title?: string;
+    howpublished?: string;
+    month?: string;
+    year?: string;
+    note?: string;
+};
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export type BibTexPHDThesis = {
+    entry: 'phdthesis';
+    author: string;
+    title: string;
+    year: string;
+    school: string;
+    address?: string;
+    month?: string;
+    keywords?: string[];
+    note?: string;
+};
+
+export type BibTexProceedings = {
+    entry: 'proceedings';
+    title: string;
+    year: string;
+    editor?: string;
+    volume?: string;
+    number?: string;
+    series?: string;
+    address?: string;
+    month?: string;
+    organization?: string;
+    publisher?: string;
+    note?: string;
+};
+
+export type BibTexTechReport = {
+    entry: 'techreport';
+    author: string;
+    title: string;
+    institution: string;
+    year: string;
+    type?: string;
+    number?: string;
+    address?: string;
+    month?: string;
+    note?: string;
+};
+
+export type BibTexUnpublished = {
+    entry: 'unpublished';
+    author: string;
+    title: string;
+    note: string;
+    month?: string;
+    year?: string;
+};
+
+export type BibTexMetadata =
+    | BibTexArticle
+    | BibTexBook
+    | BibTexBooklet
+    | BibTexConference
+    | BibTexInBook
+    | BibTexInCollection
+    | BibTexManual
+    | BibTexMastersThesis
+    | BibTexMisc
+    | BibTexPHDThesis
+    | BibTexProceedings
+    | BibTexTechReport
+    | BibTexUnpublished;
+
+type CustomMetadata = {
+    custom?: {
+        [key: string]: string;
+    };
+};
+
+export type RefMetadata = BibTexMetadata & CustomMetadata;

libs/datasource/jwt/src/types/utils.ts

@@ -0,0 +1,8 @@
+type ValueOf<T> = T[keyof T];
+export type ExcludeFunction<T> = Pick<
+    T,
+    ValueOf<{
+        // eslint-disable-next-line @typescript-eslint/ban-types
+        [K in keyof T]: T[K] extends Function ? never : K;
+    }>
+>;

libs/datasource/jwt/src/types/uuid.ts

@@ -0,0 +1,76 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+type VersionChar = '1' | '2' | '3' | '4' | '5';
+
+type Char =
+    | '0'
+    | '1'
+    | '2'
+    | '3'
+    | '4'
+    | '5'
+    | '6'
+    | '7'
+    | '8'
+    | '9'
+    | 'a'
+    | 'b'
+    | 'c'
+    | 'd'
+    | 'e'
+    | 'f';
+
+type Prev<X extends number> = [
+    never,
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    ...never[]
+][X];
+
+type HasLength<S extends string, Len extends number> = [Len] extends [0]
+    ? S extends ''
+        ? true
+        : never
+    : S extends `${infer C}${infer Rest}`
+    ? Lowercase<C> extends Char
+        ? HasLength<Rest, Prev<Len>>
+        : never
+    : never;
+
+type Char4<S extends string> = true extends HasLength<S, 4> ? S : never;
+type Char8<S extends string> = true extends HasLength<S, 8> ? S : never;
+type Char12<S extends string> = true extends HasLength<S, 12> ? S : never;
+
+type VersionGroup<S extends string> = S extends `${infer Version}${infer Rest}`
+    ? Version extends VersionChar
+        ? true extends HasLength<Rest, 3>
+            ? S
+            : never
+        : never
+    : never;
+
+type NilUUID = '00000000-0000-0000-0000-000000000000';
+export type UUID<S extends string> = S extends NilUUID
+    ? S
+    : S extends `${infer S8}-${infer S4_1}-${infer S4_2}-${infer S4_3}-${infer S12}`
+    ? S8 extends Char8<S8>
+        ? S4_1 extends Char4<S4_1>
+            ? S4_2 extends VersionGroup<S4_2>
+                ? S4_3 extends Char4<S4_3>
+                    ? S12 extends Char12<S12>
+                        ? S
+                        : never
+                    : never
+                : never
+            : never
+        : never
+    : never;

libs/datasource/jwt/src/utils/event-bus.ts

@@ -0,0 +1,107 @@
+import { debounce } from 'ts-debounce';
+
+import { getLogger } from './index';
+
+declare const JWT_DEV: boolean;
+
+const logger = getLogger('BlockDB:event_bus');
+
+export class BlockEventBus {
+    readonly #event_bus: EventTarget;
+    readonly #event_callback_cache: Map<string, any>;
+    readonly #scoped_cache: Map<string, BlockScopedEventBus<any>>;
+
+    constructor(event_bus?: EventTarget) {
+        this.#event_bus = event_bus || new EventTarget();
+        this.#event_callback_cache = new Map();
+        this.#scoped_cache = new Map();
+    }
+
+    protected on_listener(
+        topic: string,
+        name: string,
+        listener: (e: Event) => void
+    ) {
+        const handler_name = `${topic}/${name}`;
+        if (!this.#event_callback_cache.has(handler_name)) {
+            this.#event_bus.addEventListener(topic, listener);
+            this.#event_callback_cache.set(handler_name, listener);
+        } else {
+            JWT_DEV && logger(`event handler ${handler_name} is existing`);
+        }
+    }
+
+    protected off_listener(topic: string, name: string) {
+        const handler_name = `${topic}/${name}`;
+        const listener = this.#event_callback_cache.get(handler_name);
+        if (listener) {
+            this.#event_bus.removeEventListener(topic, listener);
+            this.#event_callback_cache.delete(handler_name);
+        } else {
+            JWT_DEV && logger(`event handler ${handler_name} is not existing`);
+        }
+    }
+
+    protected has_listener(topic: string, name: string) {
+        return this.#event_callback_cache.has(`${topic}/${name}`);
+    }
+
+    protected emit_event<T>(topic: string, detail?: T) {
+        this.#event_bus.dispatchEvent(new CustomEvent(topic, { detail }));
+    }
+
+    topic<T = unknown>(topic: string): BlockScopedEventBus<T> {
+        return (
+            this.#scoped_cache.get(topic) ||
+            new BlockScopedEventBus<T>(topic, this.#event_bus)
+        );
+    }
+}
+
+type DebounceOptions = {
+    wait: number;
+    maxWait?: number;
+};
+
+type ListenerOptions = {
+    debounce?: DebounceOptions;
+};
+
+class BlockScopedEventBus<T> extends BlockEventBus {
+    readonly #topic: string;
+
+    constructor(topic: string, event_bus?: EventTarget) {
+        super(event_bus);
+        this.#topic = topic;
+    }
+
+    on(
+        name: string,
+        listener: ((e: T) => Promise<void>) | ((e: T) => void),
+        options?: ListenerOptions
+    ) {
+        if (options?.debounce) {
+            const { wait, maxWait } = options.debounce;
+            const debounced = debounce(listener, wait, { maxWait });
+            this.on_listener(this.#topic, name, e => {
+                debounced((e as CustomEvent)?.detail);
+            });
+        } else {
+            this.on_listener(this.#topic, name, e => {
+                listener((e as CustomEvent)?.detail);
+            });
+        }
+    }
+
+    off(name: string) {
+        this.off_listener(this.#topic, name);
+    }
+
+    has(name: string) {
+        return this.has_listener(this.#topic, name);
+    }
+
+    emit(detail?: T) {
+        this.emit_event(this.#topic, detail);
+    }
+}