mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-15 09:06:19 +08:00
init: the first public commit for AFFiNE
This commit is contained in:
DarkSky
@@ -0,0 +1,161 @@
|
||||
import { AbstractType as YAbstractType } from 'yjs';
|
||||
|
||||
import { BlockItem } from '../types';
|
||||
|
||||
export type ChangedStateKeys = 'add' | 'update' | 'delete';
|
||||
export type ChangedStates<S = ChangedStateKeys> = Map<string, S>;
|
||||
|
||||
export type BlockListener<S = ChangedStateKeys, R = unknown> = (
|
||||
states: ChangedStates<S>
|
||||
) => Promise<R> | R;
|
||||
|
||||
export type Operable<T, Base = YAbstractType<any>> = T extends Base
|
||||
? ContentOperation
|
||||
: T;
|
||||
|
||||
export interface InternalPlainObject {}
|
||||
|
||||
export type BaseTypes = string | number | boolean | InternalPlainObject;
|
||||
|
||||
export type ContentTypes = BaseTypes | ContentOperation;
|
||||
|
||||
interface ContentOperation {
|
||||
get length(): number;
|
||||
createText(): TextOperation;
|
||||
createArray<T extends ContentTypes = ContentOperation>(): ArrayOperation<T>;
|
||||
createMap<T extends ContentTypes = ContentOperation>(): MapOperation<T>;
|
||||
asText(): TextOperation | undefined;
|
||||
asArray<T extends ContentTypes = ContentOperation>():
|
||||
| ArrayOperation<T>
|
||||
| undefined;
|
||||
asMap<T extends ContentTypes = ContentOperation>():
|
||||
| MapOperation<T>
|
||||
| undefined;
|
||||
autoGet(
|
||||
root: ThisType<ContentOperation> | Record<string, unknown>,
|
||||
path: string[]
|
||||
): unknown | undefined;
|
||||
autoSet(
|
||||
root: ThisType<ContentOperation>,
|
||||
path: string[],
|
||||
data: unknown,
|
||||
partial?: boolean
|
||||
): void;
|
||||
}
|
||||
|
||||
type TextAttributes = Record<string, string>;
|
||||
|
||||
export type TextToken = {
|
||||
insert: string;
|
||||
attributes?: TextAttributes;
|
||||
};
|
||||
|
||||
export interface TextOperation extends ContentOperation {
|
||||
insert(
|
||||
index: number,
|
||||
content: string,
|
||||
format?: Record<string, string>
|
||||
): void;
|
||||
format(index: number, length: number, format: Record<string, string>): void;
|
||||
delete(index: number, length: number): void;
|
||||
setAttribute(name: string, value: BaseTypes): void;
|
||||
getAttribute<T extends BaseTypes = string>(name: string): T | undefined;
|
||||
toString(): TextToken[];
|
||||
}
|
||||
|
||||
export interface ArrayOperation<T extends ContentTypes = ContentOperation>
|
||||
extends ContentOperation {
|
||||
insert(index: number, content: Array<Operable<T>>): void;
|
||||
delete(index: number, length: number): void;
|
||||
push(content: Array<Operable<T>>): void;
|
||||
unshift(content: Array<Operable<T>>): void;
|
||||
get(index: number): Operable<T> | undefined;
|
||||
slice(start?: number, end?: number): Array<Operable<T>>;
|
||||
map<R = unknown>(callback: (value: T, index: number) => R): Array<R>;
|
||||
forEach(callback: (value: T, index: number) => boolean | void): void;
|
||||
find<R = unknown>(
|
||||
callback: (value: T, index: number) => boolean
|
||||
): R | undefined;
|
||||
findIndex(callback: (value: T, index: number) => boolean): number;
|
||||
}
|
||||
|
||||
export interface MapOperation<T extends ContentTypes = ContentOperation>
|
||||
extends ContentOperation {
|
||||
set(key: string, value: Operable<T>): void;
|
||||
get(key: string): Operable<T> | undefined;
|
||||
delete(key: string): void;
|
||||
has(key: string): boolean;
|
||||
}
|
||||
|
||||
export type HistoryCallback<T = unknown> = (map: Map<string, T>) => void;
|
||||
|
||||
export interface HistoryManager {
|
||||
onPush<T = unknown>(name: string, callback: HistoryCallback<T>): void;
|
||||
offPush(name: string): boolean;
|
||||
onPop<T = unknown>(name: string, callback: HistoryCallback<T>): void;
|
||||
offPop(name: string): boolean;
|
||||
undo<T = unknown>(): Map<string, T> | undefined;
|
||||
redo<T = unknown>(): Map<string, T> | undefined;
|
||||
clear(): void;
|
||||
}
|
||||
|
||||
type BlockPosition = { pos?: number; before?: string; after?: string };
|
||||
|
||||
interface BlockInstance<C extends ContentOperation> {
|
||||
get id(): string;
|
||||
get type(): BlockItem<C>['type'];
|
||||
get flavor(): BlockItem<C>['flavor'];
|
||||
// TODO: flavor needs optimization
|
||||
setFlavor(flavor: BlockItem<C>['flavor']): void;
|
||||
get created(): BlockItem<C>['created'];
|
||||
get updated(): number; // update time, UTC timestamp, read only
|
||||
get creator(): string | undefined; // creator id
|
||||
get children(): string[];
|
||||
getChildren(block_ids: (string | undefined)[]): BlockInstance<C>[];
|
||||
hasChildren(block_id: string): boolean;
|
||||
insertChildren(
|
||||
block: ThisType<BlockInstance<C>>,
|
||||
pos?: BlockPosition
|
||||
): void;
|
||||
removeChildren(block_ids: (string | undefined)[]): void;
|
||||
get content(): BlockItem<C>['content'];
|
||||
on(
|
||||
key: 'children' | 'content',
|
||||
name: string,
|
||||
listener: BlockListener
|
||||
): void;
|
||||
off(key: 'children' | 'content', name: string): void;
|
||||
addChildrenListener(name: string, listener: BlockListener): void;
|
||||
removeChildrenListener(name: string): void;
|
||||
addContentListener(name: string, listener: BlockListener): void;
|
||||
removeContentListener(name: string): void;
|
||||
scopedHistory(scope: any[]): HistoryManager;
|
||||
}
|
||||
|
||||
interface AsyncDatabaseAdapter<C extends ContentOperation> {
|
||||
inspector(): Record<string, any>;
|
||||
createBlock(
|
||||
options: Pick<BlockItem<C>, 'type' | 'flavor'> & {
|
||||
binary?: ArrayBuffer;
|
||||
uuid?: string;
|
||||
}
|
||||
): Promise<BlockInstance<C>>;
|
||||
getBlock(id: string): Promise<BlockInstance<C> | undefined>;
|
||||
getBlockByFlavor(flavor: BlockItem<C>['flavor']): Promise<string[]>;
|
||||
getBlockByType(type: BlockItem<C>['type']): Promise<string[]>;
|
||||
checkBlocks(keys: string[]): Promise<boolean>;
|
||||
deleteBlocks(keys: string[]): Promise<string[]>;
|
||||
on<S, R>(key: 'editing' | 'updated', listener: BlockListener<S, R>): void;
|
||||
suspend(suspend: boolean): void;
|
||||
history(): HistoryManager;
|
||||
getUserId(): string;
|
||||
}
|
||||
|
||||
export type {
|
||||
AsyncDatabaseAdapter,
|
||||
BlockPosition,
|
||||
BlockInstance,
|
||||
ContentOperation,
|
||||
};
|
||||
export type { YjsInitOptions, YjsContentOperation } from './yjs';
|
||||
export { YjsAdapter } from './yjs';
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Array as YArray, Map as YMap } from 'yjs';
|
||||
|
||||
import { RemoteKvService } from '@toeverything/datasource/remote-kv';
|
||||
|
||||
export class YjsRemoteBinaries {
|
||||
readonly #binaries: YMap<YArray<ArrayBuffer>>; // binary instance
|
||||
readonly #remote_storage?: RemoteKvService;
|
||||
|
||||
constructor(binaries: YMap<YArray<ArrayBuffer>>, remote_token?: string) {
|
||||
this.#binaries = binaries;
|
||||
if (remote_token) {
|
||||
this.#remote_storage = new RemoteKvService(remote_token);
|
||||
} else {
|
||||
console.warn(`Remote storage is not ready`);
|
||||
}
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this.#binaries.has(name);
|
||||
}
|
||||
|
||||
async get(name: string): Promise<YArray<ArrayBuffer> | undefined> {
|
||||
if (this.#binaries.has(name)) {
|
||||
return this.#binaries.get(name);
|
||||
} else {
|
||||
// TODO: Remote Load
|
||||
try {
|
||||
const file = await this.#remote_storage?.instance.getBuffData(
|
||||
name
|
||||
);
|
||||
console.log(file);
|
||||
// return file;
|
||||
} catch (e) {
|
||||
throw new Error(`Binary ${name} not found`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async set(name: string, binary: YArray<ArrayBuffer>) {
|
||||
if (!this.#binaries.has(name)) {
|
||||
console.log(name, 'name');
|
||||
if (binary.length === 1) {
|
||||
this.#binaries.set(name, binary);
|
||||
if (this.#remote_storage) {
|
||||
// TODO: Remote Save, if there is an object with the same name remotely, the upload is skipped, because the file name is the hash of the file content
|
||||
const has_file = this.#remote_storage.instance.exist(name);
|
||||
if (!has_file) {
|
||||
const upload_file = new File(binary.toArray(), name);
|
||||
await this.#remote_storage.instance
|
||||
.upload(upload_file)
|
||||
.catch(err => {
|
||||
throw new Error(`${err} upload error`);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.warn(`Remote storage is not ready`);
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
console.log('err');
|
||||
}
|
||||
throw new Error(`Binary ${name} is invalid`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
import {
|
||||
AbstractType as YAbstractType,
|
||||
Array as YArray,
|
||||
Map as YMap,
|
||||
transact,
|
||||
} from 'yjs';
|
||||
|
||||
import { BlockInstance, BlockListener, HistoryManager } from '../index';
|
||||
import { BlockItem, BlockTypes } from '../../types';
|
||||
|
||||
import { YjsContentOperation } from './operation';
|
||||
import { ChildrenListenerHandler, ContentListenerHandler } from './listener';
|
||||
import { YjsHistoryManager } from './history';
|
||||
|
||||
const GET_BLOCK_ITEM = Symbol('GET_BLOCK_ITEM');
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const getMapFromYArray = (array: YArray<string>) =>
|
||||
new Map(array.map((child, index) => [child, index]));
|
||||
|
||||
type YjsBlockInstanceProps = {
|
||||
id: string;
|
||||
block: YMap<unknown>;
|
||||
binary?: YArray<ArrayBuffer>;
|
||||
setBlock: (
|
||||
id: string,
|
||||
block: BlockItem<YjsContentOperation>
|
||||
) => Promise<void>;
|
||||
getUpdated: (id: string) => number | undefined;
|
||||
getCreator: (id: string) => string | undefined;
|
||||
getBlockInstance: (id: string) => YjsBlockInstance | undefined;
|
||||
};
|
||||
|
||||
export class YjsBlockInstance implements BlockInstance<YjsContentOperation> {
|
||||
readonly #id: string;
|
||||
readonly #block: YMap<unknown>;
|
||||
readonly #binary?: YArray<ArrayBuffer>;
|
||||
readonly #children: YArray<string>;
|
||||
readonly #set_block: (
|
||||
id: string,
|
||||
block: BlockItem<YjsContentOperation>
|
||||
) => Promise<void>;
|
||||
readonly #get_updated: (id: string) => number | undefined;
|
||||
readonly #get_creator: (id: string) => string | undefined;
|
||||
readonly #get_block_instance: (id: string) => YjsBlockInstance | undefined;
|
||||
readonly #children_listeners: Map<string, BlockListener>;
|
||||
readonly #content_listeners: Map<string, BlockListener>;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
#children_map: Map<string, number>;
|
||||
|
||||
constructor(props: YjsBlockInstanceProps) {
|
||||
this.#id = props.id;
|
||||
this.#block = props.block;
|
||||
this.#binary = props.binary;
|
||||
|
||||
this.#children = props.block.get('children') as YArray<string>;
|
||||
this.#children_map = getMapFromYArray(this.#children);
|
||||
this.#set_block = props.setBlock;
|
||||
this.#get_updated = props.getUpdated;
|
||||
this.#get_creator = props.getCreator;
|
||||
this.#get_block_instance = props.getBlockInstance;
|
||||
|
||||
this.#children_listeners = new Map();
|
||||
this.#content_listeners = new Map();
|
||||
|
||||
const content = this.#block.get('content') as YMap<unknown>;
|
||||
|
||||
this.#children.observe(event =>
|
||||
ChildrenListenerHandler(this.#children_listeners, event)
|
||||
);
|
||||
content?.observeDeep(events =>
|
||||
ContentListenerHandler(this.#content_listeners, events)
|
||||
);
|
||||
// TODO: flavor needs optimization
|
||||
this.#block.observeDeep(events =>
|
||||
ContentListenerHandler(this.#content_listeners, events)
|
||||
);
|
||||
}
|
||||
|
||||
on(
|
||||
key: 'children' | 'content',
|
||||
name: string,
|
||||
listener: BlockListener
|
||||
): void {
|
||||
if (key === 'children') {
|
||||
this.addChildrenListener(name, listener);
|
||||
} else if (key === 'content') {
|
||||
this.addContentListener(name, listener);
|
||||
}
|
||||
}
|
||||
|
||||
off(key: 'children' | 'content', name: string): void {
|
||||
if (key === 'children') {
|
||||
this.removeChildrenListener(name);
|
||||
} else if (key === 'content') {
|
||||
this.removeContentListener(name);
|
||||
}
|
||||
}
|
||||
|
||||
addChildrenListener(name: string, listener: BlockListener): void {
|
||||
this.#children_listeners.set(name, listener);
|
||||
}
|
||||
|
||||
removeChildrenListener(name: string): void {
|
||||
this.#children_listeners.delete(name);
|
||||
}
|
||||
|
||||
addContentListener(name: string, listener: BlockListener): void {
|
||||
this.#content_listeners.set(name, listener);
|
||||
}
|
||||
|
||||
removeContentListener(name: string): void {
|
||||
this.#content_listeners.delete(name);
|
||||
}
|
||||
|
||||
get id() {
|
||||
return this.#id;
|
||||
}
|
||||
|
||||
get content(): YjsContentOperation {
|
||||
if (this.type === BlockTypes.block) {
|
||||
const content = this.#block.get('content');
|
||||
if (content instanceof YAbstractType) {
|
||||
return new YjsContentOperation(content);
|
||||
} else {
|
||||
throw new Error(`Invalid content type: ${typeof content}`);
|
||||
}
|
||||
} else if (this.type === BlockTypes.binary && this.#binary) {
|
||||
return new YjsContentOperation(this.#binary);
|
||||
}
|
||||
throw new Error(
|
||||
`Invalid content type: ${this.type}, ${this.#block.get(
|
||||
'content'
|
||||
)}, ${this.#binary}`
|
||||
);
|
||||
}
|
||||
|
||||
get type(): BlockItem<YjsContentOperation>['type'] {
|
||||
return this.#block.get(
|
||||
'type'
|
||||
) as BlockItem<YjsContentOperation>['type'];
|
||||
}
|
||||
|
||||
get flavor(): BlockItem<YjsContentOperation>['flavor'] {
|
||||
return this.#block.get(
|
||||
'flavor'
|
||||
) as BlockItem<YjsContentOperation>['flavor'];
|
||||
}
|
||||
|
||||
// TODO: bad case. Need to optimize.
|
||||
setFlavor(flavor: BlockItem<YjsContentOperation>['flavor']) {
|
||||
this.#block.set('flavor', flavor);
|
||||
}
|
||||
|
||||
get created(): BlockItem<YjsContentOperation>['created'] {
|
||||
return this.#block.get(
|
||||
'created'
|
||||
) as BlockItem<YjsContentOperation>['created'];
|
||||
}
|
||||
|
||||
get updated(): number {
|
||||
return this.#get_updated(this.#id) || this.created;
|
||||
}
|
||||
|
||||
get creator(): string | undefined {
|
||||
return this.#get_creator(this.#id);
|
||||
}
|
||||
|
||||
get children(): string[] {
|
||||
return this.#children.toArray();
|
||||
}
|
||||
|
||||
getChildren(ids?: (string | undefined)[]): YjsBlockInstance[] {
|
||||
const query_ids = ids?.filter((id): id is string => !!id) || [];
|
||||
const exists_ids = this.#children.map(id => id);
|
||||
const filter_ids = query_ids.length ? query_ids : exists_ids;
|
||||
return exists_ids
|
||||
.filter(id => filter_ids.includes(id))
|
||||
.map(id => this.#get_block_instance(id))
|
||||
.filter((v): v is YjsBlockInstance => !!v);
|
||||
}
|
||||
|
||||
hasChildren(id: string): boolean {
|
||||
if (this.children.includes(id)) return true;
|
||||
return this.getChildren().some(block => block.hasChildren(id));
|
||||
}
|
||||
|
||||
private position_calculator(
|
||||
max_pos: number,
|
||||
position?: { pos?: number; before?: string; after?: string }
|
||||
) {
|
||||
const { pos, before, after } = position || {};
|
||||
if (typeof pos === 'number' && Number.isInteger(pos)) {
|
||||
if (pos >= 0 && pos < max_pos) {
|
||||
return pos;
|
||||
}
|
||||
} else if (before) {
|
||||
const current_pos = this.#children_map.get(before || '');
|
||||
if (
|
||||
typeof current_pos === 'number' &&
|
||||
Number.isInteger(current_pos)
|
||||
) {
|
||||
const prev_pos = current_pos;
|
||||
if (prev_pos >= 0 && prev_pos < max_pos) {
|
||||
return prev_pos;
|
||||
}
|
||||
}
|
||||
} else if (after) {
|
||||
const current_pos = this.#children_map.get(after || '');
|
||||
if (
|
||||
typeof current_pos === 'number' &&
|
||||
Number.isInteger(current_pos)
|
||||
) {
|
||||
const next_pos = current_pos + 1;
|
||||
if (next_pos >= 0 && next_pos < max_pos) {
|
||||
return next_pos;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async insertChildren(
|
||||
block: YjsBlockInstance,
|
||||
pos?: { pos?: number; before?: string; after?: string }
|
||||
): Promise<void> {
|
||||
const content = block[GET_BLOCK_ITEM]();
|
||||
if (content) {
|
||||
const lastIndex = this.#children_map.get(block.id);
|
||||
if (typeof lastIndex === 'number') {
|
||||
this.#children.delete(lastIndex);
|
||||
this.#children_map = getMapFromYArray(this.#children);
|
||||
}
|
||||
|
||||
const position = this.position_calculator(
|
||||
this.#children_map.size,
|
||||
pos
|
||||
);
|
||||
if (typeof position === 'number') {
|
||||
this.#children.insert(position, [block.id]);
|
||||
} else {
|
||||
this.#children.push([block.id]);
|
||||
}
|
||||
await this.#set_block(block.id, content);
|
||||
this.#children_map = getMapFromYArray(this.#children);
|
||||
}
|
||||
}
|
||||
|
||||
removeChildren(ids: (string | undefined)[]): Promise<string[]> {
|
||||
return new Promise(resolve => {
|
||||
if (this.#children.doc) {
|
||||
transact(this.#children.doc, () => {
|
||||
const failed = [];
|
||||
for (const id of ids) {
|
||||
let idx = -1;
|
||||
for (const block_id of this.#children) {
|
||||
idx += 1;
|
||||
if (block_id === id) {
|
||||
this.#children.delete(idx);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (id) failed.push(id);
|
||||
}
|
||||
|
||||
this.#children_map = getMapFromYArray(this.#children);
|
||||
resolve(failed);
|
||||
});
|
||||
} else {
|
||||
resolve(ids.filter((id): id is string => !!id));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public scopedHistory(scope: any[]): HistoryManager {
|
||||
return new YjsHistoryManager(this.#block, scope);
|
||||
}
|
||||
|
||||
[GET_BLOCK_ITEM]() {
|
||||
// check null & undefined
|
||||
if (this.content != null) {
|
||||
return {
|
||||
type: this.type,
|
||||
flavor: this.flavor,
|
||||
children: this.#children.slice(),
|
||||
created: this.created,
|
||||
content: this.content,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { Map as YMap } from 'yjs';
|
||||
|
||||
export class GateKeeper {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
#user_id: string;
|
||||
#creators: YMap<string>;
|
||||
#common: YMap<string>;
|
||||
|
||||
constructor(userId: string, creators: YMap<string>, common: YMap<string>) {
|
||||
this.#user_id = userId;
|
||||
this.#creators = creators;
|
||||
this.#common = common;
|
||||
}
|
||||
|
||||
getCreator(block_id: string): string | undefined {
|
||||
return this.#creators.get(block_id) || this.#common.get(block_id);
|
||||
}
|
||||
|
||||
setCreator(block_id: string) {
|
||||
if (!this.#creators.get(block_id)) {
|
||||
this.#creators.set(block_id, this.#user_id);
|
||||
}
|
||||
}
|
||||
|
||||
setCommon(block_id: string) {
|
||||
if (!this.#creators.get(block_id) && !this.#common.get(block_id)) {
|
||||
this.#common.set(block_id, this.#user_id);
|
||||
}
|
||||
}
|
||||
|
||||
private check_delete(block_id: string): boolean {
|
||||
const creator = this.#creators.get(block_id);
|
||||
return creator === this.#user_id || !!this.#common.get(block_id);
|
||||
}
|
||||
|
||||
checkDeleteLists(block_ids: string[]) {
|
||||
const success = [];
|
||||
const fail = [];
|
||||
for (const block_id of block_ids) {
|
||||
if (this.check_delete(block_id)) {
|
||||
success.push(block_id);
|
||||
} else {
|
||||
fail.push(block_id);
|
||||
}
|
||||
}
|
||||
return [success, fail];
|
||||
}
|
||||
|
||||
clear() {
|
||||
this.#creators.clear();
|
||||
this.#common.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { Map as YMap, UndoManager } from 'yjs';
|
||||
|
||||
import { HistoryCallback, HistoryManager } from '../../adapter';
|
||||
|
||||
type StackItem = UndoManager['undoStack'][0];
|
||||
|
||||
export class YjsHistoryManager implements HistoryManager {
|
||||
readonly #blocks: YMap<any>;
|
||||
readonly #history_manager: UndoManager;
|
||||
readonly #push_listeners: Map<string, HistoryCallback<any>>;
|
||||
readonly #pop_listeners: Map<string, HistoryCallback<any>>;
|
||||
|
||||
constructor(scope: YMap<any>, tracker?: any[]) {
|
||||
this.#blocks = scope;
|
||||
this.#history_manager = new UndoManager(scope, {
|
||||
trackedOrigins: tracker ? new Set(tracker) : undefined,
|
||||
});
|
||||
|
||||
this.#push_listeners = new Map();
|
||||
this.#history_manager.on(
|
||||
'stack-item-added',
|
||||
(event: { stackItem: StackItem }) => {
|
||||
const meta = event.stackItem.meta;
|
||||
for (const listener of this.#push_listeners.values()) {
|
||||
listener(meta);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
this.#pop_listeners = new Map();
|
||||
this.#history_manager.on(
|
||||
'stack-item-popped',
|
||||
(event: { stackItem: StackItem }) => {
|
||||
const meta = event.stackItem.meta;
|
||||
for (const listener of this.#pop_listeners.values()) {
|
||||
listener(new Map(meta));
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
onPush<T = unknown>(name: string, callback: HistoryCallback<T>): void {
|
||||
this.#push_listeners.set(name, callback);
|
||||
}
|
||||
|
||||
offPush(name: string): boolean {
|
||||
return this.#push_listeners.delete(name);
|
||||
}
|
||||
|
||||
onPop<T = unknown>(name: string, callback: HistoryCallback<T>): void {
|
||||
this.#pop_listeners.set(name, callback);
|
||||
}
|
||||
|
||||
offPop(name: string): boolean {
|
||||
return this.#pop_listeners.delete(name);
|
||||
}
|
||||
|
||||
break(): void {
|
||||
// this.#history_manager.
|
||||
}
|
||||
|
||||
undo<T = unknown>(): Map<string, T> | undefined {
|
||||
return this.#history_manager.undo()?.meta;
|
||||
}
|
||||
|
||||
redo<T = unknown>(): Map<string, T> | undefined {
|
||||
return this.#history_manager.redo()?.meta;
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
return this.#history_manager.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,605 @@
|
||||
/* eslint-disable max-lines */
|
||||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
||||
/// <reference types="wicg-file-system-access" />
|
||||
import { Buffer } from 'buffer';
|
||||
import { saveAs } from 'file-saver';
|
||||
import { fromEvent } from 'file-selector';
|
||||
import LRUCache from 'lru-cache';
|
||||
import { debounce } from 'ts-debounce';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { IndexeddbPersistence } from 'y-indexeddb';
|
||||
import { Awareness } from 'y-protocols/awareness.js';
|
||||
import {
|
||||
Doc,
|
||||
Array as YArray,
|
||||
Map as YMap,
|
||||
transact,
|
||||
encodeStateAsUpdate,
|
||||
applyUpdate,
|
||||
} from 'yjs';
|
||||
|
||||
import { WebsocketProvider } from '@toeverything/datasource/jwt-rpc';
|
||||
|
||||
import {
|
||||
AsyncDatabaseAdapter,
|
||||
BlockListener,
|
||||
ChangedStateKeys,
|
||||
HistoryManager,
|
||||
} from '../../adapter';
|
||||
import { BucketBackend, BlockItem, BlockTypes } from '../../types';
|
||||
import { getLogger, sha3, sleep } from '../../utils';
|
||||
|
||||
import { YjsRemoteBinaries } from './binary';
|
||||
import { YjsBlockInstance } from './block';
|
||||
import { GateKeeper } from './gatekeeper';
|
||||
import {
|
||||
YjsContentOperation,
|
||||
DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_SYMBOL_INTO_INNER as INTO_INNER,
|
||||
} from './operation';
|
||||
import { EmitEvents, Suspend } from './listener';
|
||||
import { YjsHistoryManager } from './history';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
const logger = getLogger('BlockDB:yjs');
|
||||
|
||||
type YjsProviders = {
|
||||
awareness: Awareness;
|
||||
idb: IndexeddbPersistence;
|
||||
binariesIdb: IndexeddbPersistence;
|
||||
ws?: WebsocketProvider;
|
||||
backend: string;
|
||||
gatekeeper: GateKeeper;
|
||||
userId: string;
|
||||
remoteToken?: string; // remote storage token
|
||||
};
|
||||
const _yjsDatabaseInstance = new Map<string, YjsProviders>();
|
||||
|
||||
async function _initWebsocketProvider(
|
||||
url: string,
|
||||
room: string,
|
||||
doc: Doc,
|
||||
token?: string,
|
||||
params?: YjsInitOptions['params']
|
||||
): Promise<[Awareness, WebsocketProvider | undefined]> {
|
||||
const awareness = new Awareness(doc);
|
||||
if (token && !process.env['NX_FREE_LOGIN']) {
|
||||
const ws = new WebsocketProvider(token, url, room, doc, {
|
||||
awareness,
|
||||
params,
|
||||
}) as any; // TODO: type is erased after cascading references
|
||||
|
||||
// Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
|
||||
return new Promise((resolve, reject) => {
|
||||
// TODO: synced will also be triggered on reconnection after losing sync
|
||||
// There needs to be an event mechanism to emit the synchronization state to the upper layer
|
||||
ws.once('synced', () => resolve([awareness, ws]));
|
||||
ws.once('lost-connection', () => resolve([awareness, ws]));
|
||||
ws.on('connection-error', reject);
|
||||
});
|
||||
} else {
|
||||
return [awareness, undefined];
|
||||
}
|
||||
}
|
||||
|
||||
const _asyncInitLoading = new Set<string>();
|
||||
const _waitLoading = async (workspace: string) => {
|
||||
while (_asyncInitLoading.has(workspace)) {
|
||||
await sleep();
|
||||
}
|
||||
};
|
||||
|
||||
async function _initYjsDatabase(
|
||||
backend: string,
|
||||
workspace: string,
|
||||
options: {
|
||||
params: YjsInitOptions['params'];
|
||||
userId: string;
|
||||
token?: string;
|
||||
}
|
||||
): Promise<YjsProviders> {
|
||||
if (_asyncInitLoading.has(workspace)) {
|
||||
await _waitLoading(workspace);
|
||||
}
|
||||
const instance = _yjsDatabaseInstance.get(workspace);
|
||||
// tTODO:odo temporarily handle this
|
||||
if (
|
||||
instance &&
|
||||
(instance.userId === options.userId || options.userId === 'default')
|
||||
) {
|
||||
return instance;
|
||||
}
|
||||
// if (instance) return instance;
|
||||
_asyncInitLoading.add(workspace);
|
||||
const { params, userId, token: remoteToken } = options;
|
||||
|
||||
const doc = new Doc({ autoLoad: true, shouldLoad: true });
|
||||
|
||||
const idb = await new IndexeddbPersistence(workspace, doc).whenSynced;
|
||||
const [awareness, ws] = await _initWebsocketProvider(
|
||||
backend,
|
||||
workspace,
|
||||
doc,
|
||||
remoteToken,
|
||||
params
|
||||
);
|
||||
|
||||
const binaries = new Doc({ autoLoad: true, shouldLoad: true });
|
||||
const binariesIdb = await new IndexeddbPersistence(
|
||||
`${workspace}_binaries`,
|
||||
binaries
|
||||
).whenSynced;
|
||||
|
||||
const gatekeeper = new GateKeeper(
|
||||
userId,
|
||||
doc.getMap('creators'),
|
||||
doc.getMap('common')
|
||||
);
|
||||
|
||||
_yjsDatabaseInstance.set(workspace, {
|
||||
awareness,
|
||||
idb,
|
||||
binariesIdb,
|
||||
ws,
|
||||
backend,
|
||||
gatekeeper,
|
||||
userId,
|
||||
remoteToken,
|
||||
});
|
||||
_asyncInitLoading.delete(workspace);
|
||||
|
||||
return {
|
||||
awareness,
|
||||
idb,
|
||||
binariesIdb,
|
||||
ws,
|
||||
backend,
|
||||
gatekeeper,
|
||||
userId,
|
||||
remoteToken,
|
||||
};
|
||||
}
|
||||
|
||||
export type { YjsBlockInstance } from './block';
|
||||
export type { YjsContentOperation } from './operation';
|
||||
|
||||
export type YjsInitOptions = {
|
||||
backend: typeof BucketBackend[keyof typeof BucketBackend];
|
||||
params?: Record<string, string>;
|
||||
userId?: string;
|
||||
token?: string;
|
||||
};
|
||||
|
||||
export class YjsAdapter implements AsyncDatabaseAdapter<YjsContentOperation> {
|
||||
readonly #provider: YjsProviders;
|
||||
readonly #doc: Doc; // doc instance
|
||||
readonly #awareness: Awareness; // lightweight state synchronization
|
||||
readonly #gatekeeper: GateKeeper; // Simple access control
|
||||
readonly #history: YjsHistoryManager;
|
||||
|
||||
// Block Collection
|
||||
// key is a randomly generated global id
|
||||
readonly #blocks: YMap<YMap<unknown>>;
|
||||
readonly #block_updated: YMap<number>;
|
||||
// Maximum cache Block 1024, ttl 10 minutes
|
||||
readonly #block_caches: LRUCache<string, YjsBlockInstance>;
|
||||
|
||||
readonly #binaries: YjsRemoteBinaries;
|
||||
|
||||
readonly #listener: Map<string, BlockListener<any>>;
|
||||
|
||||
static async init(
|
||||
workspace: string,
|
||||
options: YjsInitOptions
|
||||
): Promise<YjsAdapter> {
|
||||
const { backend, params = {}, userId = 'default', token } = options;
|
||||
const providers = await _initYjsDatabase(backend, workspace, {
|
||||
params,
|
||||
userId,
|
||||
token,
|
||||
});
|
||||
return new YjsAdapter(providers);
|
||||
}
|
||||
|
||||
private constructor(providers: YjsProviders) {
|
||||
this.#provider = providers;
|
||||
this.#doc = providers.idb.doc;
|
||||
this.#awareness = providers.awareness;
|
||||
this.#gatekeeper = providers.gatekeeper;
|
||||
|
||||
this.#blocks = this.#doc.getMap('blocks');
|
||||
this.#block_updated = this.#doc.getMap('block_updated');
|
||||
this.#block_caches = new LRUCache({ max: 1024, ttl: 1000 * 60 * 10 });
|
||||
this.#binaries = new YjsRemoteBinaries(
|
||||
providers.binariesIdb.doc.getMap(),
|
||||
providers.remoteToken
|
||||
);
|
||||
this.#history = new YjsHistoryManager(this.#blocks);
|
||||
|
||||
this.#listener = new Map();
|
||||
|
||||
const debounced_editing_notifier = debounce(
|
||||
() => {
|
||||
const listener: BlockListener<Set<string>> | undefined =
|
||||
this.#listener.get('editing');
|
||||
if (listener) {
|
||||
const mapping = this.#awareness.getStates();
|
||||
const editing_mapping: Record<string, string[]> = {};
|
||||
for (const {
|
||||
userId,
|
||||
editing,
|
||||
updated,
|
||||
} of mapping.values()) {
|
||||
// Only return the status with refresh time within 10 seconds
|
||||
if (
|
||||
userId &&
|
||||
editing &&
|
||||
updated &&
|
||||
typeof updated === 'number' &&
|
||||
updated + 1000 * 10 > Date.now()
|
||||
) {
|
||||
if (!editing_mapping[editing])
|
||||
editing_mapping[editing] = [];
|
||||
editing_mapping[editing].push(userId);
|
||||
}
|
||||
}
|
||||
listener(
|
||||
new Map(
|
||||
Object.entries(editing_mapping).map(([k, v]) => [
|
||||
k,
|
||||
new Set(v),
|
||||
])
|
||||
)
|
||||
);
|
||||
}
|
||||
},
|
||||
200,
|
||||
{ maxWait: 1000 }
|
||||
);
|
||||
|
||||
this.#awareness.setLocalStateField('userId', providers.userId);
|
||||
|
||||
this.#awareness.on('update', debounced_editing_notifier);
|
||||
|
||||
this.#blocks.observeDeep(events => {
|
||||
const now = Date.now();
|
||||
|
||||
const keys = events.flatMap(e => {
|
||||
if ((e.path?.length | 0) > 0) {
|
||||
return [
|
||||
[e.path[0], 'update'] as [string, ChangedStateKeys],
|
||||
];
|
||||
} else {
|
||||
return Array.from(e.changes.keys.entries()).map(
|
||||
([k, { action }]) =>
|
||||
[k, action] as [string, ChangedStateKeys]
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
EmitEvents(keys, this.#listener.get('updated'));
|
||||
|
||||
transact(this.#doc, () => {
|
||||
for (const [key, action] of keys) {
|
||||
if (action === 'delete') {
|
||||
this.#block_updated.delete(key);
|
||||
} else {
|
||||
this.#block_updated.set(key, now);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
getUserId(): string {
|
||||
return this.#provider.userId;
|
||||
}
|
||||
|
||||
inspector() {
|
||||
const resolve_block = (blocks: Record<string, any>, id: string) => {
|
||||
const block = blocks[id];
|
||||
if (block) {
|
||||
return {
|
||||
...block,
|
||||
children: block.children.map((id: string) =>
|
||||
resolve_block(blocks, id)
|
||||
),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
save: () => {
|
||||
const binary = encodeStateAsUpdate(this.#doc);
|
||||
saveAs(
|
||||
new Blob([binary]),
|
||||
`affine_workspace_${new Date().toDateString()}.apk`
|
||||
);
|
||||
},
|
||||
load: async () => {
|
||||
const handles = await window.showOpenFilePicker({
|
||||
types: [
|
||||
{
|
||||
description: 'Affine Package',
|
||||
accept: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
'application/affine': ['.apk'],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const [file] = (await fromEvent(handles)) as File[];
|
||||
const binary = await file.arrayBuffer();
|
||||
await this.#provider.idb.clearData();
|
||||
const doc = new Doc({ autoLoad: true, shouldLoad: true });
|
||||
let updated = 0;
|
||||
let isUpdated = false;
|
||||
doc.on('update', () => {
|
||||
isUpdated = true;
|
||||
updated += 1;
|
||||
});
|
||||
setInterval(() => {
|
||||
if (updated > 0) updated -= 1;
|
||||
}, 500);
|
||||
|
||||
const update_check = new Promise<void>(resolve => {
|
||||
const check = async () => {
|
||||
while (!isUpdated || updated > 0) {
|
||||
await sleep();
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
check();
|
||||
});
|
||||
await new IndexeddbPersistence(this.#provider.idb.name, doc)
|
||||
.whenSynced;
|
||||
applyUpdate(doc, new Uint8Array(binary));
|
||||
await update_check;
|
||||
console.log('load success');
|
||||
},
|
||||
parse: () => this.#doc.toJSON(),
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
parse_page: (page_id: string) => {
|
||||
const blocks = this.#blocks.toJSON();
|
||||
return resolve_block(blocks, page_id);
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
parse_pages: (resolve = false) => {
|
||||
const blocks = this.#blocks.toJSON();
|
||||
return Object.fromEntries(
|
||||
Object.entries(blocks)
|
||||
.filter(([, block]) => block.flavor === 'page')
|
||||
.map(([key, block]) => {
|
||||
if (resolve) {
|
||||
return resolve_block(blocks, key);
|
||||
} else {
|
||||
return [key, block];
|
||||
}
|
||||
})
|
||||
);
|
||||
},
|
||||
clear: () => {
|
||||
this.#blocks.clear();
|
||||
this.#block_updated.clear();
|
||||
this.#gatekeeper.clear();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async createBlock(
|
||||
options: Pick<BlockItem<YjsContentOperation>, 'type' | 'flavor'> & {
|
||||
uuid?: string;
|
||||
binary?: ArrayBufferLike;
|
||||
}
|
||||
): Promise<YjsBlockInstance> {
|
||||
const uuid = options.uuid || `affine${nanoid(16)}`;
|
||||
if (options.type === BlockTypes.binary) {
|
||||
if (options.binary && options.binary instanceof ArrayBuffer) {
|
||||
const array = new YArray();
|
||||
array.insert(0, [options.binary]);
|
||||
const block = {
|
||||
type: options.type,
|
||||
flavor: options.flavor,
|
||||
children: [] as string[],
|
||||
created: Date.now(),
|
||||
content: new YjsContentOperation(array),
|
||||
hash: sha3(Buffer.from(options.binary)),
|
||||
};
|
||||
await this.set_block(uuid, block);
|
||||
return (await this.getBlock(uuid))!;
|
||||
} else {
|
||||
throw new Error(`Invalid binary type: ${options.binary}`);
|
||||
}
|
||||
} else {
|
||||
const block = {
|
||||
type: options.type,
|
||||
flavor: options.flavor,
|
||||
children: [] as string[],
|
||||
created: Date.now(),
|
||||
content: new YjsContentOperation(new YMap()),
|
||||
};
|
||||
await this.set_block(uuid, block);
|
||||
return (await this.getBlock(uuid))!;
|
||||
}
|
||||
}
|
||||
|
||||
private get_updated(id: string) {
|
||||
return this.#block_updated.get(id);
|
||||
}
|
||||
|
||||
private get_creator(id: string) {
|
||||
return this.#gatekeeper.getCreator(id);
|
||||
}
|
||||
|
||||
private get_block_sync(id: string): YjsBlockInstance | undefined {
|
||||
const cached = this.#block_caches.get(id);
|
||||
if (cached) {
|
||||
// Synchronous read cannot read binary
|
||||
if (cached.type === BlockTypes.block) {
|
||||
return cached;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const block = this.#blocks.get(id);
|
||||
|
||||
// Synchronous read cannot read binary
|
||||
if (block && block.get('type') === BlockTypes.block) {
|
||||
return new YjsBlockInstance({
|
||||
id,
|
||||
block,
|
||||
setBlock: this.set_block.bind(this),
|
||||
getUpdated: this.get_updated.bind(this),
|
||||
getCreator: this.get_creator.bind(this),
|
||||
getBlockInstance: this.get_block_sync.bind(this),
|
||||
});
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async getBlock(id: string): Promise<YjsBlockInstance | undefined> {
|
||||
const block_instance = this.get_block_sync(id);
|
||||
if (block_instance) return block_instance;
|
||||
const block = this.#blocks.get(id);
|
||||
if (block && block.get('type') === BlockTypes.binary) {
|
||||
const binary = await this.#binaries.get(
|
||||
block.get('hash') as string
|
||||
);
|
||||
if (binary) {
|
||||
return new YjsBlockInstance({
|
||||
id,
|
||||
block,
|
||||
binary,
|
||||
setBlock: this.set_block.bind(this),
|
||||
getUpdated: this.get_updated.bind(this),
|
||||
getCreator: this.get_creator.bind(this),
|
||||
getBlockInstance: this.get_block_sync.bind(this),
|
||||
});
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async getBlockByFlavor(
|
||||
flavor: BlockItem<YjsContentOperation>['flavor']
|
||||
): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
this.#blocks.forEach((doc, key) => {
|
||||
if (doc.get('flavor') === flavor) {
|
||||
keys.push(key);
|
||||
}
|
||||
});
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
async getBlockByType(
|
||||
type: BlockItem<YjsContentOperation>['type']
|
||||
): Promise<string[]> {
|
||||
const keys: string[] = [];
|
||||
this.#blocks.forEach((doc, key) => {
|
||||
if (doc.get('type') === type) {
|
||||
keys.push(key);
|
||||
}
|
||||
});
|
||||
|
||||
return keys;
|
||||
}
|
||||
|
||||
private async set_block(
|
||||
key: string,
|
||||
item: BlockItem<YjsContentOperation> & { hash?: string }
|
||||
): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const block = this.#blocks.get(key) || new YMap();
|
||||
transact(this.#doc, () => {
|
||||
// Insert only if the block doesn't exist yet
|
||||
// Other modification operations are done in the block instance
|
||||
let uploaded: Promise<void> | undefined;
|
||||
if (!block.size) {
|
||||
const content = item.content[INTO_INNER]();
|
||||
if (!content) return reject();
|
||||
|
||||
const children = new YArray();
|
||||
children.push(item.children);
|
||||
|
||||
block.set('type', item.type);
|
||||
block.set('flavor', item.flavor);
|
||||
block.set('children', children);
|
||||
block.set('created', item.created);
|
||||
if (item.type === BlockTypes.block) {
|
||||
block.set('content', content);
|
||||
} else if (item.type === BlockTypes.binary && item.hash) {
|
||||
if (content instanceof YArray) {
|
||||
block.set('hash', item.hash);
|
||||
if (!this.#binaries.has(item.hash)) {
|
||||
uploaded = this.#binaries.set(
|
||||
item.hash,
|
||||
content
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw new Error(
|
||||
'binary content must be an buffer yarray'
|
||||
);
|
||||
}
|
||||
} else {
|
||||
throw new Error('invalid block type: ' + item.type);
|
||||
}
|
||||
|
||||
this.#blocks.set(key, block);
|
||||
}
|
||||
|
||||
if (item.flavor === 'page') {
|
||||
this.#awareness.setLocalStateField('editing', key);
|
||||
this.#awareness.setLocalStateField('updated', Date.now());
|
||||
}
|
||||
// References do not add delete restrictions
|
||||
if (item.flavor === 'reference') {
|
||||
this.#gatekeeper.setCommon(key);
|
||||
} else {
|
||||
this.#gatekeeper.setCreator(key);
|
||||
}
|
||||
|
||||
if (uploaded) {
|
||||
// TODO: there should be a mechanism to retry the upload
|
||||
uploaded.catch(err => {
|
||||
// undo set on failure
|
||||
console.error('Failed to upload object: ', err);
|
||||
this.deleteBlocks([key]);
|
||||
reject(err);
|
||||
});
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async checkBlocks(keys: string[]): Promise<boolean> {
|
||||
return (
|
||||
keys.filter(key => !!this.#blocks.get(key)).length === keys.length
|
||||
);
|
||||
}
|
||||
|
||||
async deleteBlocks(keys: string[]): Promise<string[]> {
|
||||
const [success, fail] = this.#gatekeeper.checkDeleteLists(keys);
|
||||
transact(this.#doc, () => {
|
||||
for (const key of success) {
|
||||
this.#blocks.delete(key);
|
||||
}
|
||||
});
|
||||
return fail;
|
||||
}
|
||||
|
||||
on<S, R>(key: 'editing' | 'updated', listener: BlockListener<S, R>): void {
|
||||
this.#listener.set(key, listener);
|
||||
}
|
||||
|
||||
suspend(suspend: boolean) {
|
||||
Suspend(suspend);
|
||||
}
|
||||
|
||||
public history(): HistoryManager {
|
||||
return this.#history;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import { produce } from 'immer';
|
||||
import { debounce } from 'ts-debounce';
|
||||
import { YEvent } from 'yjs';
|
||||
import { BlockListener, ChangedStateKeys } from '../index';
|
||||
|
||||
let listener_suspend = false;
|
||||
|
||||
let listener_map = new Map<BlockListener, [string, ChangedStateKeys][]>();
|
||||
|
||||
const debounced_suspend_notifier = debounce(
|
||||
(listener?: BlockListener) => {
|
||||
if (listener) {
|
||||
listener_map = produce(listener_map, draft => {
|
||||
const events = draft.get(listener);
|
||||
if (events) {
|
||||
listener(new Map(events));
|
||||
draft.delete(listener);
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
500,
|
||||
{ maxWait: 2000 }
|
||||
);
|
||||
|
||||
/**
|
||||
* Suspend instant update event dispatch, extend to at least 500ms once, and up to 2000ms once when triggered continuously
|
||||
* @param suspend true: suspend monitoring, false: resume monitoring
|
||||
*/
|
||||
export function Suspend(suspend: boolean) {
|
||||
listener_suspend = produce(listener_suspend, draft => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
draft = suspend;
|
||||
});
|
||||
if (!suspend && listener_map.size) {
|
||||
listener_map = produce(listener_map, draft => {
|
||||
for (const [listener, events] of draft) {
|
||||
listener(new Map(events));
|
||||
}
|
||||
draft.clear();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function EmitEvents(
|
||||
events: [string, ChangedStateKeys][],
|
||||
listener?: BlockListener
|
||||
) {
|
||||
if (listener) {
|
||||
if (listener_suspend) {
|
||||
listener_map = produce(listener_map, draft => {
|
||||
const old_events = listener_map.get(listener) || [];
|
||||
draft.set(listener, [...old_events, ...events]);
|
||||
});
|
||||
debounced_suspend_notifier(listener);
|
||||
} else {
|
||||
listener(new Map(events));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ChildrenListenerHandler(
|
||||
listeners: Map<string, BlockListener>,
|
||||
event: YEvent<any>
|
||||
) {
|
||||
if (listeners.size) {
|
||||
const keys = Array.from(event.keys.entries()).map(
|
||||
([key, { action }]) => [key, action] as [string, ChangedStateKeys]
|
||||
);
|
||||
for (const listener of listeners.values()) {
|
||||
EmitEvents(keys, listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function ContentListenerHandler(
|
||||
listeners: Map<string, BlockListener>,
|
||||
events: YEvent<any>[]
|
||||
) {
|
||||
if (listeners.size) {
|
||||
const keys = events.flatMap(e => {
|
||||
if ((e.path?.length | 0) > 0) {
|
||||
return [[e.path[0], 'update'] as [string, 'update']];
|
||||
} else {
|
||||
return Array.from(e.changes.keys.entries()).map(
|
||||
([k, { action }]) => [k, action] as [string, typeof action]
|
||||
);
|
||||
}
|
||||
});
|
||||
if (keys.length) {
|
||||
for (const listener of listeners.values()) {
|
||||
EmitEvents(keys, listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
/* eslint-disable max-lines */
|
||||
import {
|
||||
AbstractType as YAbstractType,
|
||||
Array as YArray,
|
||||
Map as YMap,
|
||||
Text as YText,
|
||||
} from 'yjs';
|
||||
|
||||
import {
|
||||
ArrayOperation,
|
||||
BaseTypes,
|
||||
BlockListener,
|
||||
ContentOperation,
|
||||
ContentTypes,
|
||||
MapOperation,
|
||||
Operable,
|
||||
TextOperation,
|
||||
TextToken,
|
||||
} from '../index';
|
||||
import { ChildrenListenerHandler, ContentListenerHandler } from './listener';
|
||||
|
||||
const INTO_INNER = Symbol('INTO_INNER');
|
||||
|
||||
export const DO_NOT_USE_THIS_OR_YOU_WILL_BE_FIRED_SYMBOL_INTO_INNER: typeof INTO_INNER =
|
||||
INTO_INNER;
|
||||
|
||||
function auto_get(root: ContentOperation, key: string): unknown | undefined {
|
||||
const array = root.asArray();
|
||||
if (array && !Number.isNaN(Number(key))) return array.get(Number(key));
|
||||
const map = root.asMap();
|
||||
if (map) return map.get(key);
|
||||
const text = root.asText();
|
||||
if (text) return text.toString();
|
||||
console.error('auto_get unknown root', root, key);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function auto_set(root: ContentOperation, key: string, data: BaseTypes): void {
|
||||
const array = root.asArray<BaseTypes>();
|
||||
if (array && !Number.isNaN(Number(key))) {
|
||||
return array.insert(Number(key), [data]);
|
||||
}
|
||||
const map = root.asMap<BaseTypes>();
|
||||
if (map) {
|
||||
return map.set(key, data);
|
||||
}
|
||||
const text = root.asText();
|
||||
if (text && !Number.isNaN(Number(key)) && typeof data === 'string') {
|
||||
return text.insert(Number(key), data);
|
||||
}
|
||||
console.error('autoSet unknown root or path', root, key, data);
|
||||
}
|
||||
|
||||
export class YjsContentOperation implements ContentOperation {
|
||||
readonly #content: YAbstractType<unknown>;
|
||||
|
||||
constructor(content: YAbstractType<any>) {
|
||||
this.#content = content;
|
||||
}
|
||||
|
||||
get length(): number {
|
||||
if (this.#content instanceof YMap) {
|
||||
return this.#content.size;
|
||||
}
|
||||
if (this.#content instanceof YArray || this.#content instanceof YText) {
|
||||
return this.#content.length;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
createText(): YjsTextOperation {
|
||||
return new YjsTextOperation(new YText());
|
||||
}
|
||||
|
||||
createArray<
|
||||
T extends ContentTypes = ContentOperation
|
||||
>(): YjsArrayOperation<T> {
|
||||
return new YjsArrayOperation(new YArray());
|
||||
}
|
||||
|
||||
createMap<T extends ContentTypes = ContentOperation>(): YjsMapOperation<T> {
|
||||
return new YjsMapOperation(new YMap());
|
||||
}
|
||||
|
||||
asText(): YjsTextOperation | undefined {
|
||||
if (this.#content instanceof YText) {
|
||||
return new YjsTextOperation(this.#content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
asArray<T extends ContentTypes = ContentOperation>():
|
||||
| YjsArrayOperation<T>
|
||||
| undefined {
|
||||
if (this.#content instanceof YArray) {
|
||||
return new YjsArrayOperation(this.#content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
asMap<T extends ContentTypes = ContentOperation>():
|
||||
| YjsMapOperation<T>
|
||||
| undefined {
|
||||
if (this.#content instanceof YMap) {
|
||||
return new YjsMapOperation(this.#content);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
autoGet(
|
||||
root: ThisType<ContentOperation> | Record<string, unknown>,
|
||||
path: string[]
|
||||
): unknown | undefined {
|
||||
if (root) {
|
||||
if (path.length === 0) {
|
||||
return root;
|
||||
} else if (root instanceof YjsContentOperation) {
|
||||
const [key, ...rest] = path;
|
||||
const new_root = auto_get(root, key);
|
||||
if (new_root) {
|
||||
return this.autoGet(new_root as typeof root, rest);
|
||||
}
|
||||
} else if (typeof root === 'object') {
|
||||
throw new Error(
|
||||
'autoGet must not get a non-value type, this is a deprecated behavior'
|
||||
);
|
||||
}
|
||||
}
|
||||
console.error('autoGet unknown root', root, path);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
autoSet(
|
||||
root: ThisType<ContentOperation>,
|
||||
path: string[],
|
||||
data: BaseTypes,
|
||||
partial?: boolean
|
||||
): void {
|
||||
if (root) {
|
||||
if (path.length === 0) {
|
||||
if (data && typeof data === 'object') {
|
||||
throw new Error(
|
||||
'autoSet must not set a non-value type, this is a deprecated behavior'
|
||||
);
|
||||
} else {
|
||||
console.error('autoSet unknown data', root, path, data);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (root instanceof YjsContentOperation) {
|
||||
if (path.length === 1) {
|
||||
const [key] = path;
|
||||
if (key) return auto_set(root, key, data);
|
||||
console.error('autoSet unknown path', root, path, data);
|
||||
return;
|
||||
}
|
||||
const [key, ...rest] = path;
|
||||
const new_root = auto_get(root, key);
|
||||
if (new_root && new_root instanceof YjsContentOperation) {
|
||||
return this.autoSet(new_root, rest, data, partial);
|
||||
} else {
|
||||
throw new Error(
|
||||
'autoSet must not set a non-value type, this is a deprecated behavior'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.error('autoSet unknown root', root, path);
|
||||
}
|
||||
|
||||
protected into_inner<T>(content: Operable<T>): T {
|
||||
if (content instanceof YjsContentOperation) {
|
||||
return content[INTO_INNER]() as unknown as T;
|
||||
} else {
|
||||
return content as T;
|
||||
}
|
||||
}
|
||||
|
||||
protected to_operable<T>(content: T): Operable<T> {
|
||||
if (content instanceof YAbstractType) {
|
||||
return new YjsContentOperation(content) as unknown as Operable<T>;
|
||||
}
|
||||
return content as Operable<T>;
|
||||
}
|
||||
|
||||
[INTO_INNER](): YAbstractType<unknown> | undefined {
|
||||
if (this.#content instanceof YAbstractType) {
|
||||
return this.#content;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
private toJSON() {
|
||||
return this.#content.toJSON();
|
||||
}
|
||||
}
|
||||
|
||||
class YjsTextOperation extends YjsContentOperation implements TextOperation {
|
||||
readonly #content: YText;
|
||||
|
||||
constructor(content: YText) {
|
||||
super(content);
|
||||
this.#content = content;
|
||||
}
|
||||
|
||||
insert(
|
||||
index: number,
|
||||
content: string,
|
||||
format?: Record<string, string>
|
||||
): void {
|
||||
this.#content.insert(index, content, format);
|
||||
}
|
||||
|
||||
format(
|
||||
index: number,
|
||||
length: number,
|
||||
format: Record<string, string>
|
||||
): void {
|
||||
this.#content.format(index, length, format);
|
||||
}
|
||||
|
||||
delete(index: number, length: number): void {
|
||||
this.#content.delete(index, length);
|
||||
}
|
||||
|
||||
setAttribute(name: string, value: BaseTypes) {
|
||||
this.#content.setAttribute(name, value);
|
||||
}
|
||||
|
||||
getAttribute<T extends BaseTypes = string>(name: string): T | undefined {
|
||||
return this.#content.getAttribute(name);
|
||||
}
|
||||
|
||||
override toString(): TextToken[] {
|
||||
return this.#content.toDelta();
|
||||
}
|
||||
}
|
||||
|
||||
class YjsArrayOperation<T extends ContentTypes>
|
||||
extends YjsContentOperation
|
||||
implements ArrayOperation<T>
|
||||
{
|
||||
readonly #content: YArray<T>;
|
||||
readonly #listeners: Map<string, BlockListener>;
|
||||
|
||||
constructor(content: YArray<T>) {
|
||||
super(content);
|
||||
this.#content = content;
|
||||
this.#listeners = new Map();
|
||||
|
||||
this.#content.observe(event =>
|
||||
ChildrenListenerHandler(this.#listeners, event)
|
||||
);
|
||||
}
|
||||
|
||||
on(name: string, listener: BlockListener) {
|
||||
this.#listeners.set(name, listener);
|
||||
}
|
||||
|
||||
off(name: string) {
|
||||
this.#listeners.delete(name);
|
||||
}
|
||||
|
||||
insert(index: number, content: Array<Operable<T>>): void {
|
||||
this.#content.insert(
|
||||
index,
|
||||
content.map(v => this.into_inner(v))
|
||||
);
|
||||
}
|
||||
|
||||
delete(index: number, length: number): void {
|
||||
this.#content.delete(index, length);
|
||||
}
|
||||
|
||||
push(content: Array<Operable<T>>): void {
|
||||
this.#content.push(content.map(v => this.into_inner(v)));
|
||||
}
|
||||
|
||||
unshift(content: Array<Operable<T>>): void {
|
||||
this.#content.unshift(content.map(v => this.into_inner(v)));
|
||||
}
|
||||
|
||||
get(index: number): Operable<T> | undefined {
|
||||
const content = this.#content.get(index);
|
||||
if (content) return this.to_operable(content);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private get_internal(index: number): T {
|
||||
return this.#content.get(index);
|
||||
}
|
||||
|
||||
slice(start?: number, end?: number): Operable<T>[] {
|
||||
return this.#content.slice(start, end).map(v => this.to_operable(v));
|
||||
}
|
||||
|
||||
map<R = unknown>(callback: (value: T, index: number) => R): R[] {
|
||||
return this.#content.map((value, index) => callback(value, index));
|
||||
}
|
||||
|
||||
// Traverse, if callback returns false, stop traversing
|
||||
forEach(callback: (value: T, index: number) => boolean) {
|
||||
for (let i = 0; i < this.#content.length; i++) {
|
||||
const ret = callback(this.get_internal(i), i);
|
||||
if (ret === false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
find<R = unknown>(
|
||||
callback: (value: T, index: number) => boolean
|
||||
): R | undefined {
|
||||
let result: R | undefined = undefined;
|
||||
this.forEach((value, i) => {
|
||||
const found = callback(value, i);
|
||||
if (found) {
|
||||
result = value as unknown as R;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
findIndex(callback: (value: T, index: number) => boolean): number {
|
||||
let position = -1;
|
||||
this.forEach((value, i) => {
|
||||
const found = callback(value, i);
|
||||
if (found) {
|
||||
position = i;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return position;
|
||||
}
|
||||
}
|
||||
|
||||
class YjsMapOperation<T extends ContentTypes>
|
||||
extends YjsContentOperation
|
||||
implements MapOperation<T>
|
||||
{
|
||||
readonly #content: YMap<T>;
|
||||
readonly #listeners: Map<string, BlockListener>;
|
||||
|
||||
constructor(content: YMap<T>) {
|
||||
super(content);
|
||||
this.#content = content;
|
||||
this.#listeners = new Map();
|
||||
|
||||
content?.observeDeep(events =>
|
||||
ContentListenerHandler(this.#listeners, events)
|
||||
);
|
||||
}
|
||||
|
||||
on(name: string, listener: BlockListener) {
|
||||
this.#listeners.set(name, listener);
|
||||
}
|
||||
|
||||
off(name: string) {
|
||||
this.#listeners.delete(name);
|
||||
}
|
||||
|
||||
set(key: string, value: Operable<T>): void {
|
||||
if (value instanceof YjsContentOperation) {
|
||||
const content = value[INTO_INNER]();
|
||||
if (content) this.#content.set(key, content as unknown as T);
|
||||
} else {
|
||||
this.#content.set(key, value as T);
|
||||
}
|
||||
}
|
||||
|
||||
get(key: string): Operable<T> | undefined {
|
||||
const content = this.#content.get(key);
|
||||
if (content) return this.to_operable(content);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
delete(key: string): void {
|
||||
this.#content.delete(key);
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.#content.has(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
import {
|
||||
BlockInstance,
|
||||
BlockListener,
|
||||
BlockPosition,
|
||||
ContentOperation,
|
||||
ContentTypes,
|
||||
HistoryManager,
|
||||
MapOperation,
|
||||
} from '../adapter';
|
||||
import {
|
||||
BlockTypes,
|
||||
BlockTypeKeys,
|
||||
BlockFlavors,
|
||||
BlockFlavorKeys,
|
||||
} from '../types';
|
||||
import { getLogger } from '../utils';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
const logger = getLogger('BlockDB:block');
|
||||
const logger_debug = getLogger('debug:BlockDB:block');
|
||||
|
||||
const GET_BLOCK = Symbol('GET_BLOCK');
|
||||
const SET_PARENT = Symbol('SET_PARENT');
|
||||
|
||||
export class AbstractBlock<
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> {
|
||||
readonly #id: string;
|
||||
readonly #block: BlockInstance<C>;
|
||||
readonly #history: HistoryManager;
|
||||
readonly #root?: AbstractBlock<B, C>;
|
||||
readonly #parent_listener: Map<string, BlockListener>;
|
||||
|
||||
#parent?: AbstractBlock<B, C>;
|
||||
|
||||
constructor(
|
||||
block: B,
|
||||
root?: AbstractBlock<B, C>,
|
||||
parent?: AbstractBlock<B, C>
|
||||
) {
|
||||
this.#id = block.id;
|
||||
this.#block = block;
|
||||
this.#history = this.#block.scopedHistory([this.#id]);
|
||||
|
||||
this.#root = root;
|
||||
this.#parent_listener = new Map();
|
||||
this.#parent = parent;
|
||||
JWT_DEV && logger_debug(`init: exists ${this.#id}`);
|
||||
}
|
||||
|
||||
public get root() {
|
||||
return this.#root;
|
||||
}
|
||||
|
||||
protected get parent_node() {
|
||||
return this.#parent;
|
||||
}
|
||||
|
||||
protected _getParentPage(warning = true): string | undefined {
|
||||
if (this.flavor === 'page') {
|
||||
return this.#block.id;
|
||||
} else if (!this.#parent) {
|
||||
if (warning && this.flavor !== 'workspace') {
|
||||
console.warn('parent not found');
|
||||
}
|
||||
return undefined;
|
||||
} else {
|
||||
return this.#parent.parent_page;
|
||||
}
|
||||
}
|
||||
|
||||
public get parent_page(): string | undefined {
|
||||
return this._getParentPage();
|
||||
}
|
||||
|
||||
public on(
|
||||
event: 'content' | 'children' | 'parent',
|
||||
name: string,
|
||||
callback: BlockListener
|
||||
) {
|
||||
if (event === 'parent') {
|
||||
this.#parent_listener.set(name, callback);
|
||||
} else {
|
||||
this.#block.on(event, name, callback);
|
||||
}
|
||||
}
|
||||
|
||||
public off(event: 'content' | 'children' | 'parent', name: string) {
|
||||
if (event === 'parent') {
|
||||
this.#parent_listener.delete(name);
|
||||
} else {
|
||||
this.#block.off(event, name);
|
||||
}
|
||||
}
|
||||
|
||||
public addChildrenListener(name: string, listener: BlockListener) {
|
||||
this.#block.addChildrenListener(name, listener);
|
||||
}
|
||||
|
||||
public removeChildrenListener(name: string) {
|
||||
this.#block.removeChildrenListener(name);
|
||||
}
|
||||
|
||||
public addContentListener(name: string, listener: BlockListener) {
|
||||
this.#block.addContentListener(name, listener);
|
||||
}
|
||||
|
||||
public removeContentListener(name: string) {
|
||||
this.#block.removeContentListener(name);
|
||||
}
|
||||
|
||||
public getContent<
|
||||
T extends ContentTypes = ContentOperation
|
||||
>(): MapOperation<T> {
|
||||
if (this.#block.type === BlockTypes.block) {
|
||||
return this.#block.content.asMap() as MapOperation<T>;
|
||||
}
|
||||
throw new Error(
|
||||
`this block not a structured block: ${this.#id}, ${
|
||||
this.#block.type
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
public getBinary(): ArrayBuffer | undefined {
|
||||
if (this.#block.type === BlockTypes.binary) {
|
||||
return this.#block.content.asArray<ArrayBuffer>()?.get(0);
|
||||
}
|
||||
throw new Error('this block not a binary block');
|
||||
}
|
||||
|
||||
public get<R = unknown>(path: string[]): R {
|
||||
const content = this.getContent();
|
||||
return content.autoGet(content, path) as R;
|
||||
}
|
||||
|
||||
public set<V = unknown>(path: string[], value: V, partial?: boolean) {
|
||||
const content = this.getContent();
|
||||
content.autoSet(content, path, value, partial);
|
||||
}
|
||||
|
||||
private get_date_text(timestamp?: number): string | undefined {
|
||||
try {
|
||||
if (timestamp && !Number.isNaN(timestamp)) {
|
||||
return new Date(timestamp)
|
||||
.toISOString()
|
||||
.split('T')[0]
|
||||
.replace(/-/g, '');
|
||||
}
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (e) {}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Last update UTC time
|
||||
public get lastUpdated(): number {
|
||||
return this.#block.updated || this.#block.created;
|
||||
}
|
||||
|
||||
private get last_updated_date(): string | undefined {
|
||||
return this.get_date_text(this.lastUpdated);
|
||||
}
|
||||
|
||||
// create UTC time
|
||||
public get created(): number {
|
||||
return this.#block.created;
|
||||
}
|
||||
|
||||
private get created_date(): string | undefined {
|
||||
return this.get_date_text(this.created);
|
||||
}
|
||||
|
||||
// creator id
|
||||
public get creator(): string | undefined {
|
||||
return this.#block.creator;
|
||||
}
|
||||
|
||||
[GET_BLOCK]() {
|
||||
return this.#block;
|
||||
}
|
||||
|
||||
[SET_PARENT](parent: AbstractBlock<B, C>) {
|
||||
this.#parent = parent;
|
||||
const states: Map<string, 'update'> = new Map([[parent.id, 'update']]);
|
||||
for (const listener of this.#parent_listener.values()) {
|
||||
listener(states);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get document index tags
|
||||
*/
|
||||
public getTags(): string[] {
|
||||
const created = this.created_date;
|
||||
const updated = this.last_updated_date;
|
||||
|
||||
return [
|
||||
`id:${this.#id}`,
|
||||
`type:${this.type}`,
|
||||
`type:${this.flavor}`,
|
||||
this.flavor === BlockFlavors.page && `type:doc`, // normal documentation
|
||||
this.flavor === BlockFlavors.tag && `type:card`, // tag document
|
||||
// this.type === ??? && `type:theorem`, // global marked math formula
|
||||
created && `created:${created}`,
|
||||
updated && `updated:${updated}`,
|
||||
].filter((v): v is string => !!v);
|
||||
}
|
||||
|
||||
/**
|
||||
* current document instance id
|
||||
*/
|
||||
public get id(): string {
|
||||
return this.#id;
|
||||
}
|
||||
|
||||
/**
|
||||
* current block type
|
||||
*/
|
||||
public get type(): typeof BlockTypes[BlockTypeKeys] {
|
||||
return this.#block.type;
|
||||
}
|
||||
|
||||
/**
|
||||
* current block flavor
|
||||
*/
|
||||
public get flavor(): typeof BlockFlavors[BlockFlavorKeys] {
|
||||
return this.#block.flavor;
|
||||
}
|
||||
|
||||
// TODO: flavor needs optimization
|
||||
setFlavor(flavor: typeof BlockFlavors[BlockFlavorKeys]) {
|
||||
this.#block.setFlavor(flavor);
|
||||
}
|
||||
|
||||
public get children(): string[] {
|
||||
return this.#block.children;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert sub-Block
|
||||
* @param block Block instance
|
||||
* @param position Insertion position, if it is empty, it will be inserted at the end. If the block already exists, the position will be moved
|
||||
* @returns
|
||||
*/
|
||||
public async insertChildren(
|
||||
block: AbstractBlock<B, C>,
|
||||
position?: BlockPosition
|
||||
) {
|
||||
JWT_DEV && logger(`insertChildren: start`);
|
||||
|
||||
if (block.id === this.#id) return; // avoid self-reference
|
||||
if (
|
||||
this.type !== BlockTypes.block || // binary cannot insert subblocks
|
||||
(block.type !== BlockTypes.block &&
|
||||
this.flavor !== BlockFlavors.workspace) // binary can only be inserted into workspace
|
||||
) {
|
||||
throw new Error('insertChildren: binary not allow insert children');
|
||||
}
|
||||
|
||||
this.#block.insertChildren(block[GET_BLOCK](), position);
|
||||
block[SET_PARENT](this);
|
||||
}
|
||||
|
||||
public hasChildren(id: string): boolean {
|
||||
return this.#block.hasChildren(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an instance of the child Block
|
||||
* @param blockId block id
|
||||
* @returns
|
||||
*/
|
||||
protected get_children(blockId?: string): BlockInstance<C>[] {
|
||||
JWT_DEV && logger(`get children: ${blockId}`);
|
||||
return this.#block.getChildren([blockId]);
|
||||
}
|
||||
|
||||
public removeChildren(blockId?: string) {
|
||||
this.#block.removeChildren([blockId]);
|
||||
}
|
||||
|
||||
public remove() {
|
||||
JWT_DEV && logger(`remove: ${this.id}`);
|
||||
if (this.flavor !== BlockFlavors.workspace) {
|
||||
// Pages other than workspace have parents
|
||||
this.parent_node!.removeChildren(this.id);
|
||||
}
|
||||
}
|
||||
|
||||
public update(path: string[], value: Record<string, any>) {
|
||||
this.set(path, value);
|
||||
}
|
||||
|
||||
private insert_blocks(
|
||||
parentNode: AbstractBlock<B, C> | undefined,
|
||||
blocks: AbstractBlock<B, C>[],
|
||||
placement: 'before' | 'after',
|
||||
referenceNode?: AbstractBlock<B, C>
|
||||
) {
|
||||
if (!blocks || blocks.length === 0 || !parentNode) {
|
||||
return;
|
||||
}
|
||||
// TODO: array equal
|
||||
if (
|
||||
!referenceNode &&
|
||||
parentNode.children.join('') ===
|
||||
blocks.map(node => node.id).join('')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
blocks.forEach(block => {
|
||||
if (block.parent_node) {
|
||||
block.remove();
|
||||
}
|
||||
|
||||
const placement_info = {
|
||||
[placement]:
|
||||
referenceNode?.id ||
|
||||
(parentNode.hasChildNodes() &&
|
||||
parentNode.children[
|
||||
placement === 'before'
|
||||
? 0
|
||||
: parentNode.children.length - 1
|
||||
]),
|
||||
};
|
||||
|
||||
parentNode.insertChildren(
|
||||
block,
|
||||
placement_info[placement] ? placement_info : undefined
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
prepend(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this, blocks.reverse(), 'before');
|
||||
}
|
||||
|
||||
append(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this, blocks, 'after');
|
||||
}
|
||||
|
||||
before(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this.parent_node, blocks, 'before', this);
|
||||
}
|
||||
|
||||
after(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this.parent_node, blocks.reverse(), 'after', this);
|
||||
}
|
||||
|
||||
hasChildNodes() {
|
||||
return this.children.length > 0;
|
||||
}
|
||||
|
||||
hasParent(blockId?: string) {
|
||||
let parent = this.parent_node;
|
||||
while (parent) {
|
||||
if (parent.id === blockId) {
|
||||
return true;
|
||||
}
|
||||
parent = parent.parent_node;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: scoped history
|
||||
*/
|
||||
public get history(): HistoryManager {
|
||||
return this.#history;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
import {
|
||||
ArrayOperation,
|
||||
BlockInstance,
|
||||
ContentOperation,
|
||||
ContentTypes,
|
||||
InternalPlainObject,
|
||||
MapOperation,
|
||||
} from '../adapter';
|
||||
import { BlockItem } from '../types';
|
||||
import { getLogger } from '../utils';
|
||||
|
||||
import { AbstractBlock } from './abstract';
|
||||
import { BlockCapability } from './capability';
|
||||
|
||||
const logger = getLogger('BlockDB:block');
|
||||
|
||||
// TODO
|
||||
export interface Decoration extends InternalPlainObject {
|
||||
key: string;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
type Validator = <T>(value: T | undefined) => boolean | void;
|
||||
|
||||
export type IndexMetadata = Readonly<{
|
||||
content?: string;
|
||||
reference?: string;
|
||||
tags: string[];
|
||||
}>;
|
||||
export type QueryMetadata = Readonly<
|
||||
{
|
||||
[key: string]: number | string | string[] | undefined;
|
||||
} & Omit<BlockItem<any>, 'content'>
|
||||
>;
|
||||
|
||||
export type ReadableContentExporter<
|
||||
R = string,
|
||||
T extends ContentTypes = ContentOperation
|
||||
> = (content: MapOperation<T>) => R;
|
||||
type GetExporter<R> = (
|
||||
block: BlockInstance<any>
|
||||
) => Readonly<[string, ReadableContentExporter<R, any>]>[];
|
||||
|
||||
type Exporters = {
|
||||
content: GetExporter<string>;
|
||||
metadata: GetExporter<Array<[string, number | string | string[]]>>;
|
||||
tag: GetExporter<string[]>;
|
||||
};
|
||||
|
||||
export class BaseBlock<
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> extends AbstractBlock<B, C> {
|
||||
readonly #exporters?: Exporters;
|
||||
readonly #content_exporters_getter: () => Map<
|
||||
string,
|
||||
ReadableContentExporter<string, any>
|
||||
>;
|
||||
readonly #metadata_exporters_getter: () => Map<
|
||||
string,
|
||||
ReadableContentExporter<
|
||||
Array<[string, number | string | string[]]>,
|
||||
any
|
||||
>
|
||||
>;
|
||||
readonly #tag_exporters_getter: () => Map<
|
||||
string,
|
||||
ReadableContentExporter<string[], any>
|
||||
>;
|
||||
|
||||
#validators: Map<string, Validator> = new Map();
|
||||
|
||||
constructor(
|
||||
block: B,
|
||||
root?: AbstractBlock<B, C>,
|
||||
parent?: AbstractBlock<B, C>,
|
||||
exporters?: Exporters
|
||||
) {
|
||||
super(block, root, parent);
|
||||
|
||||
this.#exporters = exporters;
|
||||
this.#content_exporters_getter = () =>
|
||||
new Map(exporters?.content(block));
|
||||
this.#metadata_exporters_getter = () =>
|
||||
new Map(exporters?.metadata(block));
|
||||
this.#tag_exporters_getter = () => new Map(exporters?.tag(block));
|
||||
}
|
||||
|
||||
get parent() {
|
||||
return this.parent_node as BaseBlock<B, C> | undefined;
|
||||
}
|
||||
|
||||
private get decoration(): ArrayOperation<Decoration> | undefined {
|
||||
const content = this.getContent<ArrayOperation<Decoration>>();
|
||||
if (!content.has('decoration')) {
|
||||
const decoration = content.createArray<Decoration>();
|
||||
content.set('decoration', decoration);
|
||||
}
|
||||
return content.get('decoration')?.asArray();
|
||||
}
|
||||
|
||||
getDecoration<T = unknown>(key: string): T | undefined {
|
||||
const decoration = this.decoration?.find<Decoration>(
|
||||
decoration => decoration.key === key
|
||||
);
|
||||
if (this.validate(key, decoration?.value)) {
|
||||
return decoration?.value as T;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
getDecorations<T = Record<string, unknown>>(): T {
|
||||
const decorations = {} as T;
|
||||
this.decoration?.forEach(decoration => {
|
||||
const value = this.validate(decoration.key, decoration.value)
|
||||
? decoration.value
|
||||
: undefined;
|
||||
// @ts-ignore
|
||||
decorations[decoration.key] = value;
|
||||
});
|
||||
return decorations;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
getCapability<T extends BlockCapability>(key: string): T | undefined {
|
||||
// TODO: Capability api design
|
||||
return undefined;
|
||||
}
|
||||
|
||||
setDecoration(key: string, value: unknown) {
|
||||
if (!this.validate(key, value)) {
|
||||
throw new Error(`set [${key}] error: validate error.`);
|
||||
}
|
||||
const decoration = { key, value };
|
||||
const index =
|
||||
this.decoration?.findIndex(decoration => decoration.key === key) ??
|
||||
-1;
|
||||
if (index > -1) {
|
||||
this.decoration?.delete(index, 1);
|
||||
this.decoration?.insert(index, [decoration]);
|
||||
} else {
|
||||
this.decoration?.insert(this.decoration?.length, [decoration]);
|
||||
}
|
||||
}
|
||||
|
||||
removeDecoration(key: string) {
|
||||
const index =
|
||||
this.decoration?.findIndex(decoration => decoration.key === key) ??
|
||||
-1;
|
||||
if (index > -1) {
|
||||
this.decoration?.delete(index, 1);
|
||||
}
|
||||
}
|
||||
|
||||
clearDecoration() {
|
||||
this.decoration?.delete(0, this.decoration?.length);
|
||||
}
|
||||
|
||||
setValidator(key: string, validator?: Validator) {
|
||||
if (validator) {
|
||||
this.#validators.set(key, validator);
|
||||
} else {
|
||||
this.#validators.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
private validate(key: string, value: unknown): boolean {
|
||||
const validate = this.#validators.get(key);
|
||||
if (validate) {
|
||||
return validate(value) === false ? false : true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
get group(): BaseBlock<B, C> | undefined {
|
||||
if (this.flavor === 'group') {
|
||||
return this;
|
||||
}
|
||||
return this.parent?.group;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an instance of the child Block
|
||||
* @param blockId block id
|
||||
*/
|
||||
private get_children_instance(blockId?: string): BaseBlock<B, C>[] {
|
||||
return this.get_children(blockId).map(
|
||||
block => new BaseBlock(block, this.root, this, this.#exporters)
|
||||
);
|
||||
}
|
||||
|
||||
private get_indexable_metadata() {
|
||||
const metadata: Record<string, number | string | string[]> = {};
|
||||
for (const [name, exporter] of this.#metadata_exporters_getter()) {
|
||||
try {
|
||||
for (const [key, val] of exporter(this.getContent())) {
|
||||
metadata[key] = val;
|
||||
}
|
||||
} catch (err) {
|
||||
logger(`Failed to export metadata: ${name}`, err);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const parent_page = this._getParentPage(false);
|
||||
if (parent_page) metadata['page'] = parent_page;
|
||||
if (this.group) metadata['group'] = this.group.id;
|
||||
if (this.parent) metadata['parent'] = this.parent.id;
|
||||
} catch (e) {
|
||||
logger(`Failed to export default metadata`, e);
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
public getQueryMetadata(): QueryMetadata {
|
||||
return {
|
||||
type: this.type,
|
||||
flavor: this.flavor,
|
||||
creator: this.creator,
|
||||
children: this.children,
|
||||
created: this.created,
|
||||
updated: this.lastUpdated,
|
||||
...this.get_indexable_metadata(),
|
||||
};
|
||||
}
|
||||
|
||||
private get_indexable_content(): string | undefined {
|
||||
const contents = [];
|
||||
for (const [name, exporter] of this.#content_exporters_getter()) {
|
||||
try {
|
||||
const content = exporter(this.getContent());
|
||||
if (content) contents.push(content);
|
||||
} catch (err) {
|
||||
logger(`Failed to export content: ${name}`, err);
|
||||
}
|
||||
}
|
||||
if (!contents.length) {
|
||||
try {
|
||||
const content = this.getContent() as any;
|
||||
return JSON.stringify(content['toJSON']());
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (e) {}
|
||||
}
|
||||
return contents.join('\n');
|
||||
}
|
||||
|
||||
private get_indexable_tags(): string[] {
|
||||
const tags: string[] = [];
|
||||
for (const [name, exporter] of this.#tag_exporters_getter()) {
|
||||
try {
|
||||
tags.push(...exporter(this.getContent()));
|
||||
} catch (err) {
|
||||
logger(`Failed to export tags: ${name}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
return tags;
|
||||
}
|
||||
|
||||
public getIndexMetadata(): IndexMetadata {
|
||||
return {
|
||||
content: this.get_indexable_content(),
|
||||
reference: '', // TODO: bibliography
|
||||
tags: [...this.getTags(), ...this.get_indexable_tags()],
|
||||
};
|
||||
}
|
||||
|
||||
// ======================================
|
||||
// DOM like apis
|
||||
// ======================================
|
||||
|
||||
get firstChild() {
|
||||
return this.get_children_instance(this.children[0]);
|
||||
}
|
||||
|
||||
get lastChild() {
|
||||
const children = this.children;
|
||||
return this.get_children_instance(children[children.length - 1]);
|
||||
}
|
||||
|
||||
get nextSibling(): BaseBlock<B, C> | undefined {
|
||||
if (this.parent) {
|
||||
const parent = this.parent;
|
||||
const children = parent.children;
|
||||
const index = children.indexOf(this.id);
|
||||
return parent.get_children_instance(children[index + 1])[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
get nextSiblings(): BaseBlock<B, C>[] {
|
||||
if (this.parent) {
|
||||
const parent = this.parent;
|
||||
const children = parent.children;
|
||||
const index = children.indexOf(this.id);
|
||||
return (
|
||||
children
|
||||
.slice(index + 1)
|
||||
.flatMap(id => parent.get_children_instance(id)) || []
|
||||
);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
get previousSibling(): BaseBlock<B, C> | undefined {
|
||||
if (this.parent) {
|
||||
const parent = this.parent;
|
||||
const children = parent.children;
|
||||
const index = children.indexOf(this.id);
|
||||
return parent.get_children_instance(children[index - 1])[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
contains(block: AbstractBlock<B, C>) {
|
||||
if (this === block) {
|
||||
return true;
|
||||
}
|
||||
return this.hasChildren(block.id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { BlockSearchItem } from './index';
|
||||
|
||||
export class BlockCapability {
|
||||
// Accept a block instance, check its type, content data structure
|
||||
// Does it meet the structural requirements of the current capability
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
protected check_block(block: BlockSearchItem): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
// data structure upgrade
|
||||
protected migration(): void {
|
||||
// TODO: need to override
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { BlockMetadata } from './indexer';
|
||||
|
||||
export type BlockSearchItem = Partial<
|
||||
BlockMetadata & {
|
||||
readonly content: string;
|
||||
}
|
||||
>;
|
||||
|
||||
export { BaseBlock } from './base';
|
||||
export type { Decoration, ReadableContentExporter } from './base';
|
||||
export { BlockIndexer } from './indexer';
|
||||
export type { BlockCapability } from './capability';
|
||||
@@ -0,0 +1,316 @@
|
||||
import { deflateSync, inflateSync, strToU8, strFromU8 } from 'fflate';
|
||||
import { Document as DocumentIndexer, DocumentSearchOptions } from 'flexsearch';
|
||||
import { get, set, keys, del, createStore } from 'idb-keyval';
|
||||
import produce from 'immer';
|
||||
import LRUCache from 'lru-cache';
|
||||
import sift, { Query } from 'sift';
|
||||
|
||||
import {
|
||||
AsyncDatabaseAdapter,
|
||||
BlockInstance,
|
||||
ChangedStates,
|
||||
ContentOperation,
|
||||
} from '../adapter';
|
||||
import { BlockFlavors } from '../types';
|
||||
import { BlockEventBus, getLogger } from '../utils';
|
||||
|
||||
import { BaseBlock, IndexMetadata, QueryMetadata } from './base';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
|
||||
const logger = getLogger('BlockDB:indexing');
|
||||
const logger_debug = getLogger('debug:BlockDB:indexing');
|
||||
|
||||
type ChangedState = ChangedStates extends Map<unknown, infer R> ? R : never;
|
||||
|
||||
export type BlockMetadata = QueryMetadata & { readonly id: string };
|
||||
|
||||
function tokenizeZh(text: string) {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const tokenizer = Intl?.['v8BreakIterator'];
|
||||
if (tokenizer) {
|
||||
const it = tokenizer(['zh-CN'], { type: 'word' });
|
||||
it.adoptText(text);
|
||||
const words = [];
|
||||
|
||||
let cur = 0,
|
||||
prev = 0;
|
||||
|
||||
while (cur < text.length) {
|
||||
prev = cur;
|
||||
cur = it.next();
|
||||
words.push(text.substring(prev, cur));
|
||||
}
|
||||
|
||||
return words;
|
||||
}
|
||||
// eslint-disable-next-line no-control-regex
|
||||
return text.replace(/[\x00-\x7F]/g, '').split('');
|
||||
}
|
||||
|
||||
type IdbInstance = {
|
||||
get: (key: string) => Promise<ArrayBufferLike | undefined>;
|
||||
set: (key: string, value: ArrayBufferLike) => Promise<void>;
|
||||
keys: () => Promise<string[]>;
|
||||
delete: (key: string) => Promise<void>;
|
||||
};
|
||||
|
||||
type BlockIdbInstance = {
|
||||
index: IdbInstance;
|
||||
metadata: IdbInstance;
|
||||
};
|
||||
|
||||
function initIndexIdb(workspace: string): BlockIdbInstance {
|
||||
const index = createStore(`${workspace}_index`, 'index');
|
||||
const metadata = createStore(`${workspace}_metadata`, 'metadata');
|
||||
return {
|
||||
index: {
|
||||
get: (key: string) => get<ArrayBufferLike>(key, index),
|
||||
set: (key: string, value: ArrayBufferLike) =>
|
||||
set(key, value, index),
|
||||
keys: () => keys(index),
|
||||
delete: (key: string) => del(key, index),
|
||||
},
|
||||
metadata: {
|
||||
get: (key: string) => get<ArrayBufferLike>(key, metadata),
|
||||
set: (key: string, value: ArrayBufferLike) =>
|
||||
set(key, value, metadata),
|
||||
keys: () => keys(metadata),
|
||||
delete: (key: string) => del(key, metadata),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type BlockIndexedContent = {
|
||||
index: IndexMetadata;
|
||||
query: QueryMetadata;
|
||||
};
|
||||
|
||||
export type QueryIndexMetadata = Query<QueryMetadata>;
|
||||
|
||||
export class BlockIndexer<
|
||||
A extends AsyncDatabaseAdapter<C>,
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> {
|
||||
readonly #adapter: A;
|
||||
readonly #idb: BlockIdbInstance;
|
||||
|
||||
readonly #block_indexer: DocumentIndexer<IndexMetadata>;
|
||||
readonly #block_metadata: LRUCache<string, QueryMetadata>;
|
||||
readonly #event_bus: BlockEventBus;
|
||||
|
||||
readonly #block_builder: (
|
||||
block: BlockInstance<C>
|
||||
) => Promise<BaseBlock<B, C>>;
|
||||
|
||||
readonly #delay_index: { documents: Map<string, BaseBlock<B, C>> };
|
||||
|
||||
constructor(
|
||||
adapter: A,
|
||||
workspace: string,
|
||||
block_builder: (block: BlockInstance<C>) => Promise<BaseBlock<B, C>>,
|
||||
event_bus: BlockEventBus
|
||||
) {
|
||||
this.#adapter = adapter;
|
||||
this.#idb = initIndexIdb(workspace);
|
||||
|
||||
this.#block_indexer = new DocumentIndexer({
|
||||
document: {
|
||||
id: 'id',
|
||||
index: ['content', 'reference'],
|
||||
tag: 'tags',
|
||||
},
|
||||
encode: tokenizeZh,
|
||||
tokenize: 'forward',
|
||||
context: true,
|
||||
});
|
||||
this.#block_metadata = new LRUCache({
|
||||
max: 10240,
|
||||
ttl: 1000 * 60 * 30,
|
||||
});
|
||||
|
||||
this.#block_builder = block_builder;
|
||||
this.#event_bus = event_bus;
|
||||
|
||||
this.#delay_index = { documents: new Map() };
|
||||
|
||||
this.#event_bus
|
||||
.topic('reindex')
|
||||
.on('reindex', this.content_reindex.bind(this), {
|
||||
debounce: { wait: 1000, maxWait: 1000 * 10 },
|
||||
});
|
||||
|
||||
this.#event_bus
|
||||
.topic('save_index')
|
||||
.on('save_index', this.save_index.bind(this), {
|
||||
debounce: { wait: 1000 * 10, maxWait: 1000 * 20 },
|
||||
});
|
||||
}
|
||||
|
||||
private async content_reindex() {
|
||||
const paddings: Record<string, BlockIndexedContent> = {};
|
||||
|
||||
this.#delay_index.documents = produce(
|
||||
this.#delay_index.documents,
|
||||
draft => {
|
||||
for (const [k, block] of draft) {
|
||||
paddings[k] = {
|
||||
index: block.getIndexMetadata(),
|
||||
query: block.getQueryMetadata(),
|
||||
};
|
||||
draft.delete(k);
|
||||
}
|
||||
}
|
||||
);
|
||||
for (const [key, { index, query }] of Object.entries(paddings)) {
|
||||
if (index.content) {
|
||||
await this.#block_indexer.addAsync(key, index);
|
||||
this.#block_metadata.set(key, query);
|
||||
}
|
||||
}
|
||||
this.#event_bus.topic('save_index').emit();
|
||||
}
|
||||
|
||||
private async refresh_index(block: BaseBlock<B, C>) {
|
||||
const filter: string[] = [
|
||||
BlockFlavors.page,
|
||||
BlockFlavors.title,
|
||||
BlockFlavors.heading1,
|
||||
BlockFlavors.heading2,
|
||||
BlockFlavors.heading3,
|
||||
BlockFlavors.text,
|
||||
BlockFlavors.todo,
|
||||
BlockFlavors.reference,
|
||||
];
|
||||
if (filter.includes(block.flavor)) {
|
||||
this.#delay_index.documents = produce(
|
||||
this.#delay_index.documents,
|
||||
draft => {
|
||||
draft.set(block.id, block);
|
||||
}
|
||||
);
|
||||
|
||||
this.#event_bus.topic('reindex').emit();
|
||||
return true;
|
||||
}
|
||||
logger_debug(`skip index ${block.flavor}: ${block.id}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
async refreshIndex(id: string, state: ChangedState) {
|
||||
JWT_DEV && logger(`refreshArticleIndex: ${id}`);
|
||||
if (state === 'delete') {
|
||||
this.#delay_index.documents = produce(
|
||||
this.#delay_index.documents,
|
||||
draft => {
|
||||
this.#block_indexer.remove(id);
|
||||
this.#block_metadata.delete(id);
|
||||
draft.delete(id);
|
||||
}
|
||||
);
|
||||
return;
|
||||
}
|
||||
const block = await this.#adapter.getBlock(id);
|
||||
if (block?.id === id) {
|
||||
if (await this.refresh_index(await this.#block_builder(block))) {
|
||||
JWT_DEV &&
|
||||
logger(
|
||||
state
|
||||
? `refresh index: ${id}, ${state}`
|
||||
: `indexing: ${id}`
|
||||
);
|
||||
} else {
|
||||
JWT_DEV && logger(`skip index: ${id}, ${block.flavor}`);
|
||||
}
|
||||
} else {
|
||||
JWT_DEV && logger(`refreshArticleIndex: ${id} not exists`);
|
||||
}
|
||||
}
|
||||
|
||||
async loadIndex() {
|
||||
for (const key of await this.#idb.index.keys()) {
|
||||
const content = await this.#idb.index.get(key);
|
||||
if (content) {
|
||||
const decoded = strFromU8(inflateSync(new Uint8Array(content)));
|
||||
try {
|
||||
await this.#block_indexer.import(key, decoded as any);
|
||||
} catch (e) {
|
||||
console.error(`Failed to load index ${key}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of await this.#idb.metadata.keys()) {
|
||||
const content = await this.#idb.metadata.get(key);
|
||||
if (content) {
|
||||
const decoded = strFromU8(inflateSync(new Uint8Array(content)));
|
||||
try {
|
||||
await this.#block_indexer.import(key, JSON.parse(decoded));
|
||||
} catch (e) {
|
||||
console.error(`Failed to load index ${key}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(this.#block_metadata.keys());
|
||||
}
|
||||
|
||||
private async save_index() {
|
||||
const idb = this.#idb;
|
||||
await idb.index
|
||||
.keys()
|
||||
.then(keys => Promise.all(keys.map(key => idb.index.delete(key))));
|
||||
await this.#block_indexer.export((key, data) => {
|
||||
return idb.index.set(
|
||||
String(key),
|
||||
deflateSync(strToU8(data as any))
|
||||
);
|
||||
});
|
||||
const metadata = this.#block_metadata;
|
||||
await idb.metadata
|
||||
.keys()
|
||||
.then(keys =>
|
||||
Promise.all(
|
||||
keys
|
||||
.filter(key => !metadata.has(key))
|
||||
.map(key => idb.metadata.delete(key))
|
||||
)
|
||||
);
|
||||
for (const [key, data] of metadata.entries()) {
|
||||
await idb.metadata.set(
|
||||
key,
|
||||
deflateSync(strToU8(JSON.stringify(data)))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async inspectIndex() {
|
||||
const index: Record<string | number, any> = {};
|
||||
await this.#block_indexer.export((key, data) => {
|
||||
index[key] = data;
|
||||
});
|
||||
}
|
||||
|
||||
public search(
|
||||
part_of_title_or_content:
|
||||
| string
|
||||
| Partial<DocumentSearchOptions<boolean>>
|
||||
) {
|
||||
return this.#block_indexer.search(part_of_title_or_content as string);
|
||||
}
|
||||
|
||||
public query(query: QueryIndexMetadata) {
|
||||
const matches: string[] = [];
|
||||
const filter = sift<QueryMetadata>(query);
|
||||
this.#block_metadata.forEach((value, key) => {
|
||||
if (filter(value)) matches.push(key);
|
||||
});
|
||||
return matches;
|
||||
}
|
||||
|
||||
public getMetadata(ids: string[]): Array<BlockMetadata> {
|
||||
return ids
|
||||
.filter(id => this.#block_metadata.has(id))
|
||||
.map(id => ({ ...this.#block_metadata.get(id)!, id }));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,592 @@
|
||||
/* eslint-disable max-lines */
|
||||
import { DocumentSearchOptions } from 'flexsearch';
|
||||
import LRUCache from 'lru-cache';
|
||||
|
||||
import {
|
||||
AsyncDatabaseAdapter,
|
||||
YjsAdapter,
|
||||
YjsInitOptions,
|
||||
YjsContentOperation,
|
||||
ChangedStates,
|
||||
BlockListener,
|
||||
BlockInstance,
|
||||
ContentOperation,
|
||||
HistoryManager,
|
||||
ContentTypes,
|
||||
} from './adapter';
|
||||
import { YjsBlockInstance } from './adapter/yjs';
|
||||
import {
|
||||
BaseBlock,
|
||||
BlockIndexer,
|
||||
BlockSearchItem,
|
||||
ReadableContentExporter,
|
||||
} from './block';
|
||||
import { QueryIndexMetadata } from './block/indexer';
|
||||
import {
|
||||
BlockTypes,
|
||||
BlockTypeKeys,
|
||||
BlockFlavors,
|
||||
BucketBackend,
|
||||
UUID,
|
||||
BlockFlavorKeys,
|
||||
BlockItem,
|
||||
ExcludeFunction,
|
||||
} from './types';
|
||||
import { BlockEventBus, genUUID, getLogger } from './utils';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
|
||||
const logger = getLogger('BlockDB:client');
|
||||
// const logger_debug = getLogger('debug:BlockDB:client');
|
||||
|
||||
const namedUuid = Symbol('namedUUID');
|
||||
|
||||
type BlockUuid<T extends string> = T extends UUID<T> ? T : never;
|
||||
type BlockUuidOrType<T extends string> = T extends
|
||||
| BlockTypeKeys
|
||||
| BlockFlavorKeys
|
||||
? T
|
||||
: T extends string
|
||||
? BlockUuid<T>
|
||||
: never;
|
||||
|
||||
type BlockInstanceValue = ExcludeFunction<BlockInstance<any>>;
|
||||
export type BlockMatcher = Partial<BlockInstanceValue>;
|
||||
|
||||
type BlockExporters<R> = Map<
|
||||
string,
|
||||
[BlockMatcher, ReadableContentExporter<R, any>]
|
||||
>;
|
||||
type BlockClientOptions = {
|
||||
content?: BlockExporters<string>;
|
||||
metadata?: BlockExporters<Array<[string, number | string | string[]]>>;
|
||||
tagger?: BlockExporters<string[]>;
|
||||
};
|
||||
|
||||
export class BlockClient<
|
||||
A extends AsyncDatabaseAdapter<C>,
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> {
|
||||
readonly #adapter: A;
|
||||
readonly #workspace: string;
|
||||
|
||||
// Maximum cache Block 8192, ttl 30 minutes
|
||||
readonly #block_caches: LRUCache<string, BaseBlock<B, C>>;
|
||||
readonly #block_indexer: BlockIndexer<A, B, C>;
|
||||
|
||||
readonly #exporters: {
|
||||
readonly content: BlockExporters<string>;
|
||||
readonly metadata: BlockExporters<
|
||||
Array<[string, number | string | string[]]>
|
||||
>;
|
||||
readonly tag: BlockExporters<string[]>;
|
||||
};
|
||||
|
||||
readonly #event_bus: BlockEventBus;
|
||||
|
||||
readonly #parent_mapping: Map<string, string[]>;
|
||||
readonly #page_mapping: Map<string, string>;
|
||||
|
||||
readonly #root: { node?: BaseBlock<B, C> };
|
||||
|
||||
private constructor(
|
||||
adapter: A,
|
||||
workspace: string,
|
||||
options?: BlockClientOptions
|
||||
) {
|
||||
this.#adapter = adapter;
|
||||
this.#workspace = workspace;
|
||||
|
||||
this.#block_caches = new LRUCache({ max: 8192, ttl: 1000 * 60 * 30 });
|
||||
|
||||
this.#exporters = {
|
||||
content: options?.content || new Map(),
|
||||
metadata: options?.metadata || new Map(),
|
||||
tag: options?.tagger || new Map(),
|
||||
};
|
||||
|
||||
this.#event_bus = new BlockEventBus();
|
||||
|
||||
this.#block_indexer = new BlockIndexer(
|
||||
this.#adapter,
|
||||
this.#workspace,
|
||||
this.block_builder.bind(this),
|
||||
this.#event_bus.topic('indexer')
|
||||
);
|
||||
|
||||
this.#parent_mapping = new Map();
|
||||
this.#page_mapping = new Map();
|
||||
this.#adapter.on('editing', (states: ChangedStates) =>
|
||||
this.#event_bus.topic('editing').emit(states)
|
||||
);
|
||||
this.#adapter.on('updated', (states: ChangedStates) =>
|
||||
this.#event_bus.topic('updated').emit(states)
|
||||
);
|
||||
|
||||
this.#event_bus
|
||||
.topic<string[]>('rebuild_index')
|
||||
.on('rebuild_index', this.rebuild_index.bind(this), {
|
||||
debounce: { wait: 1000, maxWait: 1000 },
|
||||
});
|
||||
|
||||
this.#root = {};
|
||||
}
|
||||
|
||||
public addBlockListener(tag: string, listener: BlockListener) {
|
||||
const bus = this.#event_bus.topic<ChangedStates>('updated');
|
||||
if (tag !== 'index' || !bus.has(tag)) bus.on(tag, listener);
|
||||
else console.error(`block listener ${tag} is reserved`);
|
||||
}
|
||||
|
||||
public removeBlockListener(tag: string) {
|
||||
this.#event_bus.topic('updated').off(tag);
|
||||
}
|
||||
|
||||
public addEditingListener(
|
||||
tag: string,
|
||||
listener: BlockListener<Set<string>>
|
||||
) {
|
||||
const bus =
|
||||
this.#event_bus.topic<ChangedStates<Set<string>>>('editing');
|
||||
if (tag !== 'index' || !bus.has(tag)) bus.on(tag, listener);
|
||||
else console.error(`editing listener ${tag} is reserved`);
|
||||
}
|
||||
|
||||
public removeEditingListener(tag: string) {
|
||||
this.#event_bus.topic('editing').off(tag);
|
||||
}
|
||||
|
||||
private inspector() {
|
||||
return {
|
||||
...this.#adapter.inspector(),
|
||||
indexed: () => this.#block_indexer.inspectIndex(),
|
||||
};
|
||||
}
|
||||
|
||||
private async rebuild_index(exists_ids?: string[]) {
|
||||
JWT_DEV && logger(`rebuild index`);
|
||||
const blocks = await this.#adapter.getBlockByType(BlockTypes.block);
|
||||
const excluded = exists_ids || [];
|
||||
await Promise.all(
|
||||
blocks
|
||||
.filter(id => !excluded.includes(id))
|
||||
.map(id => this.#block_indexer.refreshIndex(id, 'add'))
|
||||
);
|
||||
}
|
||||
|
||||
public async buildIndex() {
|
||||
JWT_DEV && logger(`buildIndex: start`);
|
||||
// Skip the block index that exists in the metadata, assuming that the index of the block existing in the metadata is the latest, and modify this part if there is a problem
|
||||
// Although there may be cases where the index is refreshed but the metadata is not refreshed, re-indexing will be automatically triggered after the block is changed
|
||||
const exists_ids = await this.#block_indexer.loadIndex();
|
||||
await this.rebuild_index(exists_ids);
|
||||
this.addBlockListener('index', async states => {
|
||||
await Promise.allSettled(
|
||||
Array.from(states.entries()).map(([id, state]) => {
|
||||
if (state === 'delete') this.#block_caches.delete(id);
|
||||
return this.#block_indexer.refreshIndex(id, state);
|
||||
})
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific type of block, currently only the article type is supported
|
||||
* @param block_type block type
|
||||
* @returns
|
||||
*/
|
||||
public async getByType(
|
||||
block_type: BlockTypeKeys | BlockFlavorKeys
|
||||
): Promise<Map<string, BaseBlock<B, C>>> {
|
||||
JWT_DEV && logger(`getByType: ${block_type}`);
|
||||
const ids = [
|
||||
...this.#block_indexer.query({
|
||||
type: BlockTypes[block_type as BlockTypeKeys],
|
||||
}),
|
||||
...this.#block_indexer.query({
|
||||
flavor: BlockFlavors[block_type as BlockFlavorKeys],
|
||||
}),
|
||||
];
|
||||
const docs = await Promise.all(
|
||||
ids.map(id =>
|
||||
this.get(id as BlockUuidOrType<typeof id>).then(
|
||||
doc => [id, doc] as const
|
||||
)
|
||||
)
|
||||
);
|
||||
return new Map(docs.filter(([, doc]) => doc.children.length));
|
||||
}
|
||||
|
||||
/**
|
||||
* research all
|
||||
* @param part_of_title_or_content Title or content keyword, support Chinese
|
||||
* @param part_of_title_or_content.index search range, optional values: title, ttl, content, reference
|
||||
* @param part_of_title_or_content.tag tag, string or array of strings, supports multiple tags
|
||||
* @param part_of_title_or_content.query keyword, support Chinese
|
||||
* @param part_of_title_or_content.limit The limit of the number of search results, the default is 100
|
||||
* @param part_of_title_or_content.offset search result offset, used for page turning, default is 0
|
||||
* @param part_of_title_or_content.suggest Fuzzy matching, after enabling the content including some keywords can also be searched, the default is false
|
||||
* @returns array of search results, each array is a list of attributed block ids
|
||||
*/
|
||||
public search(
|
||||
part_of_title_or_content:
|
||||
| string
|
||||
| Partial<DocumentSearchOptions<boolean>>
|
||||
) {
|
||||
return this.#block_indexer.search(part_of_title_or_content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Full text search, the returned results are grouped by page dimension
|
||||
* @param part_of_title_or_content Title or content keyword, support Chinese
|
||||
* @param part_of_title_or_content.index search range, optional values: title, ttl, content, reference
|
||||
* @param part_of_title_or_content.tag tag, string or array of strings, supports multiple tags
|
||||
* @param part_of_title_or_content.query keyword, support Chinese
|
||||
* @param part_of_title_or_content.limit The limit of the number of search results, the default is 100
|
||||
* @param part_of_title_or_content.offset search result offset, used for page turning, default is 0
|
||||
* @param part_of_title_or_content.suggest Fuzzy matching, after enabling the content including some keywords can also be searched, the default is false
|
||||
* @returns array of search results, each array is a page
|
||||
*/
|
||||
public async searchPages(
|
||||
part_of_title_or_content:
|
||||
| string
|
||||
| Partial<DocumentSearchOptions<boolean>>
|
||||
): Promise<BlockSearchItem[]> {
|
||||
const promised_pages = await Promise.all(
|
||||
this.search(part_of_title_or_content).flatMap(({ result }) =>
|
||||
result.map(async id => {
|
||||
const page = this.#page_mapping.get(id as string);
|
||||
if (page) return page;
|
||||
const block = await this.get(id as BlockTypeKeys);
|
||||
return this.set_page(block);
|
||||
})
|
||||
)
|
||||
);
|
||||
const pages = [
|
||||
...new Set(promised_pages.filter((v): v is string => !!v)),
|
||||
];
|
||||
return Promise.all(
|
||||
this.#block_indexer.getMetadata(pages).map(async page => ({
|
||||
content: this.get_decoded_content(
|
||||
await this.#adapter.getBlock(page.id)
|
||||
),
|
||||
...page,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inquire
|
||||
* @returns array of search results
|
||||
*/
|
||||
public query(query: QueryIndexMetadata): string[] {
|
||||
return this.#block_indexer.query(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a fixed name, which has the same UUID in each workspace, and is automatically created when it does not exist
|
||||
* Generally used to store workspace-level global configuration
|
||||
* @param name block name
|
||||
* @returns block instance
|
||||
*/
|
||||
private async get_named_block(
|
||||
name: string,
|
||||
options?: { workspace?: boolean }
|
||||
): Promise<BaseBlock<B, C>> {
|
||||
const block = await this.get(genUUID(name), {
|
||||
flavor: options?.workspace
|
||||
? BlockFlavors.workspace
|
||||
: BlockFlavors.page,
|
||||
[namedUuid]: true,
|
||||
});
|
||||
return block;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the workspace block of the current instance
|
||||
* @returns block instance
|
||||
*/
|
||||
public async getWorkspace() {
|
||||
if (!this.#root.node) {
|
||||
this.#root.node = await this.get_named_block(this.#workspace, {
|
||||
workspace: true,
|
||||
});
|
||||
}
|
||||
return this.#root.node;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated custom data including access to ws configuration is unified through baseBlock.get/setDecoration(key).
|
||||
* - Get the config of the workspace block of the current instance
|
||||
* @returns MapOperation
|
||||
*/
|
||||
public async getWorkspaceConfig<T = unknown>() {
|
||||
return (await this.getWorkspace()).getContent<T>();
|
||||
}
|
||||
|
||||
private async get_parent(id: string) {
|
||||
const parents = this.#parent_mapping.get(id);
|
||||
if (parents) {
|
||||
const parent_block_id = parents[0];
|
||||
if (!this.#block_caches.has(parent_block_id)) {
|
||||
this.#block_caches.set(
|
||||
parent_block_id,
|
||||
await this.get(parent_block_id as BlockTypeKeys)
|
||||
);
|
||||
}
|
||||
return this.#block_caches.get(parent_block_id);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private set_parent(parent: string, child: string) {
|
||||
const parents = this.#parent_mapping.get(child);
|
||||
if (parents?.length) {
|
||||
if (!parents.includes(parent)) {
|
||||
console.error('parent already exists', child, parents);
|
||||
this.#parent_mapping.set(child, [...parents, parent]);
|
||||
}
|
||||
} else {
|
||||
this.#parent_mapping.set(child, [parent]);
|
||||
}
|
||||
}
|
||||
|
||||
private set_page(block: BaseBlock<B, C>) {
|
||||
const page = this.#page_mapping.get(block.id);
|
||||
if (page) return page;
|
||||
const parent_page = block.parent_page;
|
||||
if (parent_page) {
|
||||
this.#page_mapping.set(block.id, parent_page);
|
||||
return parent_page;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
registerContentExporter<T extends ContentTypes>(
|
||||
name: string,
|
||||
matcher: BlockMatcher,
|
||||
exporter: ReadableContentExporter<string, T>
|
||||
) {
|
||||
this.#exporters.content.set(name, [matcher, exporter]);
|
||||
this.#event_bus.topic('rebuild_index').emit(); // // rebuild the index every time the content exporter is registered
|
||||
}
|
||||
|
||||
unregisterContentExporter(name: string) {
|
||||
this.#exporters.content.delete(name);
|
||||
this.#event_bus.topic('rebuild_index').emit(); // Rebuild indexes every time content exporter logs out
|
||||
}
|
||||
|
||||
registerMetadataExporter<T extends ContentTypes>(
|
||||
name: string,
|
||||
matcher: BlockMatcher,
|
||||
exporter: ReadableContentExporter<
|
||||
Array<[string, number | string | string[]]>,
|
||||
T
|
||||
>
|
||||
) {
|
||||
this.#exporters.metadata.set(name, [matcher, exporter]);
|
||||
this.#event_bus.topic('rebuild_index').emit(); // // rebuild the index every time the content exporter is registered
|
||||
}
|
||||
|
||||
unregisterMetadataExporter(name: string) {
|
||||
this.#exporters.metadata.delete(name);
|
||||
this.#event_bus.topic('rebuild_index').emit(); // Rebuild indexes every time content exporter logs out
|
||||
}
|
||||
|
||||
registerTagExporter<T extends ContentTypes>(
|
||||
name: string,
|
||||
matcher: BlockMatcher,
|
||||
exporter: ReadableContentExporter<string[], T>
|
||||
) {
|
||||
this.#exporters.tag.set(name, [matcher, exporter]);
|
||||
this.#event_bus.topic('rebuild_index').emit(); // Reindex every tag exporter registration
|
||||
}
|
||||
|
||||
unregisterTagExporter(name: string) {
|
||||
this.#exporters.tag.delete(name);
|
||||
this.#event_bus.topic('rebuild_index').emit(); // Reindex every time tag exporter logs out
|
||||
}
|
||||
|
||||
private get_exporters<R>(
|
||||
exporter_map: BlockExporters<R>,
|
||||
block: BlockInstance<C>
|
||||
): Readonly<[string, ReadableContentExporter<R, any>]>[] {
|
||||
const exporters = [];
|
||||
for (const [name, [cond, exporter]] of exporter_map) {
|
||||
const conditions = Object.entries(cond);
|
||||
let matched = 0;
|
||||
for (const [key, value] of conditions) {
|
||||
if (block[key as keyof BlockInstanceValue] === value) {
|
||||
matched += 1;
|
||||
}
|
||||
}
|
||||
if (matched === conditions.length)
|
||||
exporters.push([name, exporter] as const);
|
||||
}
|
||||
|
||||
return exporters;
|
||||
}
|
||||
|
||||
private get_decoded_content(block?: BlockInstance<C>) {
|
||||
if (block) {
|
||||
const [exporter] = this.get_exporters(
|
||||
this.#exporters.content,
|
||||
block
|
||||
);
|
||||
if (exporter) {
|
||||
const op = block.content.asMap();
|
||||
if (op) return exporter[1](op);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private async block_builder(
|
||||
block: BlockInstance<C>,
|
||||
root?: BaseBlock<B, C>
|
||||
) {
|
||||
return new BaseBlock(
|
||||
block,
|
||||
root,
|
||||
(await this.get_parent(block.id)) || root,
|
||||
{
|
||||
content: block =>
|
||||
this.get_exporters(this.#exporters.content, block),
|
||||
metadata: block =>
|
||||
this.get_exporters(this.#exporters.metadata, block),
|
||||
tag: block => this.get_exporters(this.#exporters.tag, block),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a Block, which is automatically created if it does not exist
|
||||
* @param block_id_or_type block id, create a new text block when BlockTypes/BlockFlavors are not provided, does not exist or is provided. If BlockTypes/BlockFlavors are provided, create a block of the corresponding type
|
||||
* @param options.type The type of block created when block does not exist, the default is block
|
||||
* @param options.flavor The flavor of the block created when the block does not exist, the default is text
|
||||
* @param options.binary content of binary block, must be provided when type or block_id_or_type is binary
|
||||
* @returns block instance
|
||||
*/
|
||||
public async get<S extends string>(
|
||||
block_id_or_type?: BlockUuidOrType<S>,
|
||||
options?: {
|
||||
type?: BlockItem<C>['type'];
|
||||
flavor: BlockItem<C>['flavor'];
|
||||
binary?: ArrayBuffer;
|
||||
[namedUuid]?: boolean;
|
||||
}
|
||||
): Promise<BaseBlock<B, C>> {
|
||||
JWT_DEV && logger(`get: ${block_id_or_type}`);
|
||||
const {
|
||||
type = BlockTypes.block,
|
||||
flavor = BlockFlavors.text,
|
||||
binary,
|
||||
[namedUuid]: is_named_uuid,
|
||||
} = options || {};
|
||||
if (block_id_or_type && this.#block_caches.has(block_id_or_type)) {
|
||||
return this.#block_caches.get(block_id_or_type) as BaseBlock<B, C>;
|
||||
} else {
|
||||
const block =
|
||||
(block_id_or_type &&
|
||||
(await this.#adapter.getBlock(block_id_or_type))) ||
|
||||
(await this.#adapter.createBlock({
|
||||
uuid: is_named_uuid ? block_id_or_type : undefined,
|
||||
binary,
|
||||
type:
|
||||
block_id_or_type &&
|
||||
BlockTypes[block_id_or_type as BlockTypeKeys]
|
||||
? BlockTypes[block_id_or_type as BlockTypeKeys]
|
||||
: type,
|
||||
flavor:
|
||||
block_id_or_type &&
|
||||
BlockFlavors[block_id_or_type as BlockFlavorKeys]
|
||||
? BlockFlavors[block_id_or_type as BlockFlavorKeys]
|
||||
: flavor,
|
||||
}));
|
||||
|
||||
const root = is_named_uuid ? undefined : await this.getWorkspace();
|
||||
|
||||
for (const child of block.children) {
|
||||
this.set_parent(block.id, child);
|
||||
}
|
||||
|
||||
const abstract_block = await this.block_builder(block, root);
|
||||
this.set_page(abstract_block);
|
||||
abstract_block.on('parent', 'client_hook', state => {
|
||||
const [parent] = state.keys();
|
||||
this.set_parent(parent, abstract_block.id);
|
||||
this.set_page(abstract_block);
|
||||
});
|
||||
this.#block_caches.set(abstract_block.id, abstract_block);
|
||||
return abstract_block;
|
||||
}
|
||||
}
|
||||
|
||||
public async getBlockByFlavor(
|
||||
flavor: BlockItem<C>['flavor']
|
||||
): Promise<string[]> {
|
||||
return await this.#adapter.getBlockByFlavor(flavor);
|
||||
}
|
||||
|
||||
public getUserId(): string {
|
||||
return this.#adapter.getUserId();
|
||||
}
|
||||
|
||||
public has(block_ids: string[]): Promise<boolean> {
|
||||
return this.#adapter.checkBlocks(block_ids);
|
||||
}
|
||||
|
||||
/**
|
||||
* Suspend instant update event dispatch, extend to a maximum of 500ms once, and a maximum of 2000ms when triggered continuously
|
||||
* @param suspend true: suspend monitoring, false: resume monitoring
|
||||
*/
|
||||
suspend(suspend: boolean) {
|
||||
this.#adapter.suspend(suspend);
|
||||
}
|
||||
|
||||
public get history(): HistoryManager {
|
||||
return this.#adapter.history();
|
||||
}
|
||||
|
||||
public static async init(
|
||||
workspace: string,
|
||||
options: Partial<YjsInitOptions & BlockClientOptions> = {}
|
||||
): Promise<BlockClientInstance> {
|
||||
const instance = await YjsAdapter.init(workspace, {
|
||||
backend: BucketBackend.YjsWebSocketAffine,
|
||||
...options,
|
||||
});
|
||||
return new BlockClient(instance, workspace, options);
|
||||
}
|
||||
}
|
||||
|
||||
export type BlockImplInstance = BaseBlock<
|
||||
YjsBlockInstance,
|
||||
YjsContentOperation
|
||||
>;
|
||||
export type BlockClientInstance = BlockClient<
|
||||
YjsAdapter,
|
||||
YjsBlockInstance,
|
||||
YjsContentOperation
|
||||
>;
|
||||
|
||||
export type BlockInitOptions = NonNullable<
|
||||
Parameters<typeof BlockClient.init>[1]
|
||||
>;
|
||||
|
||||
export type {
|
||||
TextOperation,
|
||||
ArrayOperation,
|
||||
MapOperation,
|
||||
ChangedStates,
|
||||
} from './adapter';
|
||||
export type {
|
||||
BlockSearchItem,
|
||||
Decoration as BlockDecoration,
|
||||
ReadableContentExporter as BlockContentExporter,
|
||||
} from './block';
|
||||
export type { BlockTypeKeys } from './types';
|
||||
export { BlockTypes, BucketBackend as BlockBackend } from './types';
|
||||
export { isBlock } from './utils';
|
||||
export type { QueryIndexMetadata };
|
||||
@@ -0,0 +1,81 @@
|
||||
import { ContentOperation } from '../adapter';
|
||||
import { RefMetadata } from './metadata';
|
||||
import { UUID } from './uuid';
|
||||
// base type of block
|
||||
// y_block: tree structure expression with YAbstractType as root
|
||||
// y_binary: arbitrary binary data
|
||||
export const BlockTypes = {
|
||||
block: 'y_block' as const, // data block
|
||||
binary: 'y_binary' as const, // binary data
|
||||
};
|
||||
|
||||
// block flavor
|
||||
// block has the same basic structure
|
||||
// But different flavors provide different parsing of their content
|
||||
// TODO, how do blockdb BlockFlavors synchronize with the protocol of '@toeverything/components/editor-blocks'?
|
||||
export const BlockFlavors = {
|
||||
workspace: 'workspace' as const, // workspace
|
||||
page: 'page' as const, // page
|
||||
group: 'group' as const, // group
|
||||
title: 'title' as const, // title
|
||||
text: 'text' as const, // text
|
||||
heading1: 'heading1' as const, // heading 1
|
||||
heading2: 'heading2' as const, // heading 2
|
||||
heading3: 'heading3' as const, // heading 3
|
||||
code: 'code' as const, // code block
|
||||
todo: 'todo' as const, // todo
|
||||
numbered: 'numbered' as const, // numbered list
|
||||
bullet: 'bullet' as const, // bullet list
|
||||
comments: 'comments' as const, // comments
|
||||
tag: 'tag' as const, // tag
|
||||
reference: 'reference' as const, // reference
|
||||
image: 'image' as const, // image
|
||||
file: 'file' as const, //file
|
||||
audio: 'audio' as const, // audio
|
||||
video: 'video' as const, // video
|
||||
shape: 'shape' as const, // artboard shape
|
||||
quote: 'quote' as const, // quote
|
||||
toc: 'toc' as const, //directory
|
||||
database: 'database' as const, //Multidimensional table
|
||||
whiteboard: 'whiteboard' as const, // whiteboard
|
||||
template: 'template' as const, // template
|
||||
discussion: 'discussion' as const, // comment header
|
||||
comment: 'comment' as const, // comment details
|
||||
activity: 'activity' as const, // dynamic message
|
||||
divider: 'divider' as const, // divider line
|
||||
groupDivider: 'groupDivider' as const, // group divider
|
||||
youtube: 'youtube' as const, // Youtube
|
||||
figma: 'figma' as const, // figma
|
||||
embedLink: 'embedLink' as const, //embed link
|
||||
toggle: 'toggle' as const, // collapse the list
|
||||
callout: 'callout' as const, // reminder
|
||||
grid: 'grid' as const, // grid layout
|
||||
gridItem: 'gridItem' as const, // grid layout children
|
||||
};
|
||||
|
||||
export type BlockTypeKeys = keyof typeof BlockTypes;
|
||||
export type BlockFlavorKeys = keyof typeof BlockFlavors;
|
||||
|
||||
export type BlockItem<C extends ContentOperation> = {
|
||||
readonly type: typeof BlockTypes[BlockTypeKeys];
|
||||
flavor: typeof BlockFlavors[BlockFlavorKeys];
|
||||
children: string[];
|
||||
readonly created: number; // creation time, UTC timestamp
|
||||
readonly updated?: number; // update time, UTC timestamp
|
||||
readonly creator?: string; // creator id
|
||||
content: C; // Essentially what is stored here is either Uint8Array (binary resource) or YDoc (structured resource)
|
||||
};
|
||||
|
||||
export type BlockPage = {
|
||||
title?: string;
|
||||
description?: string;
|
||||
// preview
|
||||
preview?: UUID<'00000000-0000-0000-0000-000000000000'>;
|
||||
// Whether it is a draft, the draft will not be indexed
|
||||
draft?: string;
|
||||
// Expiration time, documents that exceed this time will be deleted
|
||||
ttl?: number;
|
||||
// Metadata, currently only bibtex definitions
|
||||
// When metadata exists, it will be treated as a reference and can be searched by tag reference
|
||||
metadata?: RefMetadata;
|
||||
};
|
||||
@@ -0,0 +1,28 @@
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
export { BlockTypes, BlockFlavors } from './block';
|
||||
export type { BlockTypeKeys, BlockFlavorKeys, BlockItem } from './block';
|
||||
export type { UUID } from './uuid';
|
||||
export type { ExcludeFunction } from './utils';
|
||||
|
||||
function getLocation() {
|
||||
try {
|
||||
const { protocol, host } = window.location;
|
||||
return { protocol, host };
|
||||
} catch (e) {
|
||||
return { protocol: 'http:', host: 'localhost' };
|
||||
}
|
||||
}
|
||||
|
||||
function getCollaborationPoint() {
|
||||
const { protocol, host } = getLocation();
|
||||
const ws = protocol.startsWith('https') ? 'wss' : 'ws';
|
||||
const isOnline = host.endsWith('affine.pro');
|
||||
const site = isOnline ? host : 'localhost:4200';
|
||||
return `${ws}://${site}/collaboration/`;
|
||||
}
|
||||
|
||||
export const BucketBackend = {
|
||||
IndexedDB: 'idb',
|
||||
WebSQL: 'websql',
|
||||
YjsWebSocketAffine: getCollaborationPoint(),
|
||||
};
|
||||
@@ -0,0 +1,203 @@
|
||||
export type BibTexArticle = {
|
||||
entry: 'article';
|
||||
author: string;
|
||||
title: string;
|
||||
journal: string;
|
||||
year: string;
|
||||
volume?: string;
|
||||
number?: string;
|
||||
pages?: string;
|
||||
month?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexBook = {
|
||||
entry: 'book';
|
||||
author: string;
|
||||
editor?: string;
|
||||
title: string;
|
||||
publisher: string;
|
||||
year: string;
|
||||
volume?: string;
|
||||
number?: string;
|
||||
series?: string;
|
||||
address?: string;
|
||||
edition?: string;
|
||||
month?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexBooklet = {
|
||||
entry: 'booklet';
|
||||
title: string;
|
||||
author?: string;
|
||||
howpublished?: string;
|
||||
address?: string;
|
||||
month?: string;
|
||||
year?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexConference = {
|
||||
entry: 'conference' | 'inproceedings';
|
||||
author: string;
|
||||
title: string;
|
||||
booktitle: string;
|
||||
year: string;
|
||||
editor?: string;
|
||||
volume?: string;
|
||||
number?: string;
|
||||
series?: string;
|
||||
pages?: string;
|
||||
address?: string;
|
||||
month?: string;
|
||||
organization?: string;
|
||||
publisher?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexInBook = {
|
||||
entry: 'inbook';
|
||||
author: string;
|
||||
editor: string;
|
||||
title: string;
|
||||
chapter: string;
|
||||
pages?: string;
|
||||
publisher: string;
|
||||
year: string;
|
||||
volume?: string;
|
||||
number?: string;
|
||||
series?: string;
|
||||
type?: string;
|
||||
address?: string;
|
||||
edition?: string;
|
||||
month?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexInCollection = {
|
||||
entry: 'incollection';
|
||||
author: string;
|
||||
title: string;
|
||||
booktitle: string;
|
||||
publisher: string;
|
||||
year: string;
|
||||
editor?: string;
|
||||
volume?: string;
|
||||
number?: string;
|
||||
series?: string;
|
||||
type?: string;
|
||||
chapter?: string;
|
||||
pages?: string;
|
||||
address?: string;
|
||||
edition?: string;
|
||||
month?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexManual = {
|
||||
entry: 'manual';
|
||||
title: string;
|
||||
author?: string;
|
||||
organization?: string;
|
||||
address?: string;
|
||||
edition?: string;
|
||||
month?: string;
|
||||
year?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexMastersThesis = {
|
||||
entry: 'mastersthesis';
|
||||
author: string;
|
||||
title: string;
|
||||
school: string;
|
||||
year: string;
|
||||
type?: string;
|
||||
address?: string;
|
||||
month?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexMisc = {
|
||||
entry: 'misc';
|
||||
author?: string;
|
||||
title?: string;
|
||||
howpublished?: string;
|
||||
month?: string;
|
||||
year?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
export type BibTexPHDThesis = {
|
||||
entry: 'phdthesis';
|
||||
author: string;
|
||||
title: string;
|
||||
year: string;
|
||||
school: string;
|
||||
address?: string;
|
||||
month?: string;
|
||||
keywords?: string[];
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexProceedings = {
|
||||
entry: 'proceedings';
|
||||
title: string;
|
||||
year: string;
|
||||
editor?: string;
|
||||
volume?: string;
|
||||
number?: string;
|
||||
series?: string;
|
||||
address?: string;
|
||||
month?: string;
|
||||
organization?: string;
|
||||
publisher?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexTechReport = {
|
||||
entry: 'techreport';
|
||||
author: string;
|
||||
title: string;
|
||||
institution: string;
|
||||
year: string;
|
||||
type?: string;
|
||||
number?: string;
|
||||
address?: string;
|
||||
month?: string;
|
||||
note?: string;
|
||||
};
|
||||
|
||||
export type BibTexUnpublished = {
|
||||
entry: 'unpublished';
|
||||
author: string;
|
||||
title: string;
|
||||
note: string;
|
||||
month?: string;
|
||||
year?: string;
|
||||
};
|
||||
|
||||
export type BibTexMetadata =
|
||||
| BibTexArticle
|
||||
| BibTexBook
|
||||
| BibTexBooklet
|
||||
| BibTexConference
|
||||
| BibTexInBook
|
||||
| BibTexInCollection
|
||||
| BibTexManual
|
||||
| BibTexMastersThesis
|
||||
| BibTexMisc
|
||||
| BibTexPHDThesis
|
||||
| BibTexProceedings
|
||||
| BibTexTechReport
|
||||
| BibTexUnpublished;
|
||||
|
||||
type CustomMetadata = {
|
||||
custom?: {
|
||||
[key: string]: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type RefMetadata = BibTexMetadata & CustomMetadata;
|
||||
@@ -0,0 +1,8 @@
|
||||
type ValueOf<T> = T[keyof T];
|
||||
export type ExcludeFunction<T> = Pick<
|
||||
T,
|
||||
ValueOf<{
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
[K in keyof T]: T[K] extends Function ? never : K;
|
||||
}>
|
||||
>;
|
||||
@@ -0,0 +1,76 @@
|
||||
/* eslint-disable @typescript-eslint/naming-convention */
|
||||
type VersionChar = '1' | '2' | '3' | '4' | '5';
|
||||
|
||||
type Char =
|
||||
| '0'
|
||||
| '1'
|
||||
| '2'
|
||||
| '3'
|
||||
| '4'
|
||||
| '5'
|
||||
| '6'
|
||||
| '7'
|
||||
| '8'
|
||||
| '9'
|
||||
| 'a'
|
||||
| 'b'
|
||||
| 'c'
|
||||
| 'd'
|
||||
| 'e'
|
||||
| 'f';
|
||||
|
||||
type Prev<X extends number> = [
|
||||
never,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
...never[]
|
||||
][X];
|
||||
|
||||
type HasLength<S extends string, Len extends number> = [Len] extends [0]
|
||||
? S extends ''
|
||||
? true
|
||||
: never
|
||||
: S extends `${infer C}${infer Rest}`
|
||||
? Lowercase<C> extends Char
|
||||
? HasLength<Rest, Prev<Len>>
|
||||
: never
|
||||
: never;
|
||||
|
||||
type Char4<S extends string> = true extends HasLength<S, 4> ? S : never;
|
||||
type Char8<S extends string> = true extends HasLength<S, 8> ? S : never;
|
||||
type Char12<S extends string> = true extends HasLength<S, 12> ? S : never;
|
||||
|
||||
type VersionGroup<S extends string> = S extends `${infer Version}${infer Rest}`
|
||||
? Version extends VersionChar
|
||||
? true extends HasLength<Rest, 3>
|
||||
? S
|
||||
: never
|
||||
: never
|
||||
: never;
|
||||
|
||||
type NilUUID = '00000000-0000-0000-0000-000000000000';
|
||||
export type UUID<S extends string> = S extends NilUUID
|
||||
? S
|
||||
: S extends `${infer S8}-${infer S4_1}-${infer S4_2}-${infer S4_3}-${infer S12}`
|
||||
? S8 extends Char8<S8>
|
||||
? S4_1 extends Char4<S4_1>
|
||||
? S4_2 extends VersionGroup<S4_2>
|
||||
? S4_3 extends Char4<S4_3>
|
||||
? S12 extends Char12<S12>
|
||||
? S
|
||||
: never
|
||||
: never
|
||||
: never
|
||||
: never
|
||||
: never
|
||||
: never;
|
||||
@@ -0,0 +1,107 @@
|
||||
import { debounce } from 'ts-debounce';
|
||||
|
||||
import { getLogger } from './index';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
|
||||
const logger = getLogger('BlockDB:event_bus');
|
||||
|
||||
export class BlockEventBus {
|
||||
readonly #event_bus: EventTarget;
|
||||
readonly #event_callback_cache: Map<string, any>;
|
||||
readonly #scoped_cache: Map<string, BlockScopedEventBus<any>>;
|
||||
|
||||
constructor(event_bus?: EventTarget) {
|
||||
this.#event_bus = event_bus || new EventTarget();
|
||||
this.#event_callback_cache = new Map();
|
||||
this.#scoped_cache = new Map();
|
||||
}
|
||||
|
||||
protected on_listener(
|
||||
topic: string,
|
||||
name: string,
|
||||
listener: (e: Event) => void
|
||||
) {
|
||||
const handler_name = `${topic}/${name}`;
|
||||
if (!this.#event_callback_cache.has(handler_name)) {
|
||||
this.#event_bus.addEventListener(topic, listener);
|
||||
this.#event_callback_cache.set(handler_name, listener);
|
||||
} else {
|
||||
JWT_DEV && logger(`event handler ${handler_name} is existing`);
|
||||
}
|
||||
}
|
||||
|
||||
protected off_listener(topic: string, name: string) {
|
||||
const handler_name = `${topic}/${name}`;
|
||||
const listener = this.#event_callback_cache.get(handler_name);
|
||||
if (listener) {
|
||||
this.#event_bus.removeEventListener(topic, listener);
|
||||
this.#event_callback_cache.delete(handler_name);
|
||||
} else {
|
||||
JWT_DEV && logger(`event handler ${handler_name} is not existing`);
|
||||
}
|
||||
}
|
||||
|
||||
protected has_listener(topic: string, name: string) {
|
||||
return this.#event_callback_cache.has(`${topic}/${name}`);
|
||||
}
|
||||
|
||||
protected emit_event<T>(topic: string, detail?: T) {
|
||||
this.#event_bus.dispatchEvent(new CustomEvent(topic, { detail }));
|
||||
}
|
||||
|
||||
topic<T = unknown>(topic: string): BlockScopedEventBus<T> {
|
||||
return (
|
||||
this.#scoped_cache.get(topic) ||
|
||||
new BlockScopedEventBus<T>(topic, this.#event_bus)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
type DebounceOptions = {
|
||||
wait: number;
|
||||
maxWait?: number;
|
||||
};
|
||||
|
||||
type ListenerOptions = {
|
||||
debounce?: DebounceOptions;
|
||||
};
|
||||
|
||||
class BlockScopedEventBus<T> extends BlockEventBus {
|
||||
readonly #topic: string;
|
||||
|
||||
constructor(topic: string, event_bus?: EventTarget) {
|
||||
super(event_bus);
|
||||
this.#topic = topic;
|
||||
}
|
||||
|
||||
on(
|
||||
name: string,
|
||||
listener: ((e: T) => Promise<void>) | ((e: T) => void),
|
||||
options?: ListenerOptions
|
||||
) {
|
||||
if (options?.debounce) {
|
||||
const { wait, maxWait } = options.debounce;
|
||||
const debounced = debounce(listener, wait, { maxWait });
|
||||
this.on_listener(this.#topic, name, e => {
|
||||
debounced((e as CustomEvent)?.detail);
|
||||
});
|
||||
} else {
|
||||
this.on_listener(this.#topic, name, e => {
|
||||
listener((e as CustomEvent)?.detail);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
off(name: string) {
|
||||
this.off_listener(this.#topic, name);
|
||||
}
|
||||
|
||||
has(name: string) {
|
||||
return this.has_listener(this.#topic, name);
|
||||
}
|
||||
|
||||
emit(detail?: T) {
|
||||
this.emit_event(this.#topic, detail);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Buffer } from 'buffer';
|
||||
import debug from 'debug';
|
||||
import { enableAllPlugins } from 'immer';
|
||||
import { SHAKE } from 'sha3';
|
||||
import { v5 as UUIDv5 } from 'uuid';
|
||||
import { AbstractBlock } from '../block/abstract';
|
||||
|
||||
import { UUID } from '../types';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
|
||||
enableAllPlugins();
|
||||
|
||||
const hash = new SHAKE(128);
|
||||
|
||||
// sha3-256(toeverythinguuid) -> truncate 128 bits
|
||||
// e66a34f77a3b09d2020eb20e1f77e3c56250c19788ed2c70993ad2c495e55de6
|
||||
const UUID_NAMESPACE = Uint8Array.from([
|
||||
0xe6, 0x6a, 0x34, 0xf7, 0x7a, 0x3b, 0x09, 0xd2, 0x02, 0x0e, 0xb2, 0x0e,
|
||||
0x1f, 0x77, 0xe3, 0xc5,
|
||||
]);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
export function genUUID(workspace: string): UUID<string> {
|
||||
return UUIDv5(workspace, UUID_NAMESPACE) as UUID<string>;
|
||||
}
|
||||
|
||||
export function sha3(buffer: Buffer): string {
|
||||
hash.reset();
|
||||
hash.update(buffer);
|
||||
return hash
|
||||
.digest('base64')
|
||||
.replace(/=/g, '')
|
||||
.replace(/\+/g, '-')
|
||||
.replace(/\//g, '_');
|
||||
}
|
||||
|
||||
export function getLogger(namespace: string) {
|
||||
if (JWT_DEV) {
|
||||
const logger = debug(namespace);
|
||||
logger.log = console.log.bind(console);
|
||||
if (JWT_DEV === ('testing' as any)) logger.enabled = true;
|
||||
return logger;
|
||||
} else {
|
||||
return () => {};
|
||||
}
|
||||
}
|
||||
|
||||
export function isBlock(obj: any) {
|
||||
return obj && obj instanceof AbstractBlock;
|
||||
}
|
||||
|
||||
export function sleep() {
|
||||
return new Promise(resolve => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
export { BlockEventBus } from './event-bus';
|
||||
Reference in New Issue
Block a user