mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-02-14 21:27:20 +00:00
init: the first public commit for AFFiNE
This commit is contained in:
DarkSky
372
libs/datasource/jwt/src/block/abstract.ts
Normal file
372
libs/datasource/jwt/src/block/abstract.ts
Normal file
@@ -0,0 +1,372 @@
|
||||
import {
|
||||
BlockInstance,
|
||||
BlockListener,
|
||||
BlockPosition,
|
||||
ContentOperation,
|
||||
ContentTypes,
|
||||
HistoryManager,
|
||||
MapOperation,
|
||||
} from '../adapter';
|
||||
import {
|
||||
BlockTypes,
|
||||
BlockTypeKeys,
|
||||
BlockFlavors,
|
||||
BlockFlavorKeys,
|
||||
} from '../types';
|
||||
import { getLogger } from '../utils';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
const logger = getLogger('BlockDB:block');
|
||||
const logger_debug = getLogger('debug:BlockDB:block');
|
||||
|
||||
const GET_BLOCK = Symbol('GET_BLOCK');
|
||||
const SET_PARENT = Symbol('SET_PARENT');
|
||||
|
||||
export class AbstractBlock<
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> {
|
||||
readonly #id: string;
|
||||
readonly #block: BlockInstance<C>;
|
||||
readonly #history: HistoryManager;
|
||||
readonly #root?: AbstractBlock<B, C>;
|
||||
readonly #parent_listener: Map<string, BlockListener>;
|
||||
|
||||
#parent?: AbstractBlock<B, C>;
|
||||
|
||||
constructor(
|
||||
block: B,
|
||||
root?: AbstractBlock<B, C>,
|
||||
parent?: AbstractBlock<B, C>
|
||||
) {
|
||||
this.#id = block.id;
|
||||
this.#block = block;
|
||||
this.#history = this.#block.scopedHistory([this.#id]);
|
||||
|
||||
this.#root = root;
|
||||
this.#parent_listener = new Map();
|
||||
this.#parent = parent;
|
||||
JWT_DEV && logger_debug(`init: exists ${this.#id}`);
|
||||
}
|
||||
|
||||
public get root() {
|
||||
return this.#root;
|
||||
}
|
||||
|
||||
protected get parent_node() {
|
||||
return this.#parent;
|
||||
}
|
||||
|
||||
protected _getParentPage(warning = true): string | undefined {
|
||||
if (this.flavor === 'page') {
|
||||
return this.#block.id;
|
||||
} else if (!this.#parent) {
|
||||
if (warning && this.flavor !== 'workspace') {
|
||||
console.warn('parent not found');
|
||||
}
|
||||
return undefined;
|
||||
} else {
|
||||
return this.#parent.parent_page;
|
||||
}
|
||||
}
|
||||
|
||||
public get parent_page(): string | undefined {
|
||||
return this._getParentPage();
|
||||
}
|
||||
|
||||
public on(
|
||||
event: 'content' | 'children' | 'parent',
|
||||
name: string,
|
||||
callback: BlockListener
|
||||
) {
|
||||
if (event === 'parent') {
|
||||
this.#parent_listener.set(name, callback);
|
||||
} else {
|
||||
this.#block.on(event, name, callback);
|
||||
}
|
||||
}
|
||||
|
||||
public off(event: 'content' | 'children' | 'parent', name: string) {
|
||||
if (event === 'parent') {
|
||||
this.#parent_listener.delete(name);
|
||||
} else {
|
||||
this.#block.off(event, name);
|
||||
}
|
||||
}
|
||||
|
||||
public addChildrenListener(name: string, listener: BlockListener) {
|
||||
this.#block.addChildrenListener(name, listener);
|
||||
}
|
||||
|
||||
public removeChildrenListener(name: string) {
|
||||
this.#block.removeChildrenListener(name);
|
||||
}
|
||||
|
||||
public addContentListener(name: string, listener: BlockListener) {
|
||||
this.#block.addContentListener(name, listener);
|
||||
}
|
||||
|
||||
public removeContentListener(name: string) {
|
||||
this.#block.removeContentListener(name);
|
||||
}
|
||||
|
||||
public getContent<
|
||||
T extends ContentTypes = ContentOperation
|
||||
>(): MapOperation<T> {
|
||||
if (this.#block.type === BlockTypes.block) {
|
||||
return this.#block.content.asMap() as MapOperation<T>;
|
||||
}
|
||||
throw new Error(
|
||||
`this block not a structured block: ${this.#id}, ${
|
||||
this.#block.type
|
||||
}`
|
||||
);
|
||||
}
|
||||
|
||||
public getBinary(): ArrayBuffer | undefined {
|
||||
if (this.#block.type === BlockTypes.binary) {
|
||||
return this.#block.content.asArray<ArrayBuffer>()?.get(0);
|
||||
}
|
||||
throw new Error('this block not a binary block');
|
||||
}
|
||||
|
||||
public get<R = unknown>(path: string[]): R {
|
||||
const content = this.getContent();
|
||||
return content.autoGet(content, path) as R;
|
||||
}
|
||||
|
||||
public set<V = unknown>(path: string[], value: V, partial?: boolean) {
|
||||
const content = this.getContent();
|
||||
content.autoSet(content, path, value, partial);
|
||||
}
|
||||
|
||||
private get_date_text(timestamp?: number): string | undefined {
|
||||
try {
|
||||
if (timestamp && !Number.isNaN(timestamp)) {
|
||||
return new Date(timestamp)
|
||||
.toISOString()
|
||||
.split('T')[0]
|
||||
.replace(/-/g, '');
|
||||
}
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (e) {}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Last update UTC time
|
||||
public get lastUpdated(): number {
|
||||
return this.#block.updated || this.#block.created;
|
||||
}
|
||||
|
||||
private get last_updated_date(): string | undefined {
|
||||
return this.get_date_text(this.lastUpdated);
|
||||
}
|
||||
|
||||
// create UTC time
|
||||
public get created(): number {
|
||||
return this.#block.created;
|
||||
}
|
||||
|
||||
private get created_date(): string | undefined {
|
||||
return this.get_date_text(this.created);
|
||||
}
|
||||
|
||||
// creator id
|
||||
public get creator(): string | undefined {
|
||||
return this.#block.creator;
|
||||
}
|
||||
|
||||
[GET_BLOCK]() {
|
||||
return this.#block;
|
||||
}
|
||||
|
||||
[SET_PARENT](parent: AbstractBlock<B, C>) {
|
||||
this.#parent = parent;
|
||||
const states: Map<string, 'update'> = new Map([[parent.id, 'update']]);
|
||||
for (const listener of this.#parent_listener.values()) {
|
||||
listener(states);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get document index tags
|
||||
*/
|
||||
public getTags(): string[] {
|
||||
const created = this.created_date;
|
||||
const updated = this.last_updated_date;
|
||||
|
||||
return [
|
||||
`id:${this.#id}`,
|
||||
`type:${this.type}`,
|
||||
`type:${this.flavor}`,
|
||||
this.flavor === BlockFlavors.page && `type:doc`, // normal documentation
|
||||
this.flavor === BlockFlavors.tag && `type:card`, // tag document
|
||||
// this.type === ??? && `type:theorem`, // global marked math formula
|
||||
created && `created:${created}`,
|
||||
updated && `updated:${updated}`,
|
||||
].filter((v): v is string => !!v);
|
||||
}
|
||||
|
||||
/**
|
||||
* current document instance id
|
||||
*/
|
||||
public get id(): string {
|
||||
return this.#id;
|
||||
}
|
||||
|
||||
/**
|
||||
* current block type
|
||||
*/
|
||||
public get type(): typeof BlockTypes[BlockTypeKeys] {
|
||||
return this.#block.type;
|
||||
}
|
||||
|
||||
/**
|
||||
* current block flavor
|
||||
*/
|
||||
public get flavor(): typeof BlockFlavors[BlockFlavorKeys] {
|
||||
return this.#block.flavor;
|
||||
}
|
||||
|
||||
// TODO: flavor needs optimization
|
||||
setFlavor(flavor: typeof BlockFlavors[BlockFlavorKeys]) {
|
||||
this.#block.setFlavor(flavor);
|
||||
}
|
||||
|
||||
public get children(): string[] {
|
||||
return this.#block.children;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert sub-Block
|
||||
* @param block Block instance
|
||||
* @param position Insertion position, if it is empty, it will be inserted at the end. If the block already exists, the position will be moved
|
||||
* @returns
|
||||
*/
|
||||
public async insertChildren(
|
||||
block: AbstractBlock<B, C>,
|
||||
position?: BlockPosition
|
||||
) {
|
||||
JWT_DEV && logger(`insertChildren: start`);
|
||||
|
||||
if (block.id === this.#id) return; // avoid self-reference
|
||||
if (
|
||||
this.type !== BlockTypes.block || // binary cannot insert subblocks
|
||||
(block.type !== BlockTypes.block &&
|
||||
this.flavor !== BlockFlavors.workspace) // binary can only be inserted into workspace
|
||||
) {
|
||||
throw new Error('insertChildren: binary not allow insert children');
|
||||
}
|
||||
|
||||
this.#block.insertChildren(block[GET_BLOCK](), position);
|
||||
block[SET_PARENT](this);
|
||||
}
|
||||
|
||||
public hasChildren(id: string): boolean {
|
||||
return this.#block.hasChildren(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an instance of the child Block
|
||||
* @param blockId block id
|
||||
* @returns
|
||||
*/
|
||||
protected get_children(blockId?: string): BlockInstance<C>[] {
|
||||
JWT_DEV && logger(`get children: ${blockId}`);
|
||||
return this.#block.getChildren([blockId]);
|
||||
}
|
||||
|
||||
public removeChildren(blockId?: string) {
|
||||
this.#block.removeChildren([blockId]);
|
||||
}
|
||||
|
||||
public remove() {
|
||||
JWT_DEV && logger(`remove: ${this.id}`);
|
||||
if (this.flavor !== BlockFlavors.workspace) {
|
||||
// Pages other than workspace have parents
|
||||
this.parent_node!.removeChildren(this.id);
|
||||
}
|
||||
}
|
||||
|
||||
public update(path: string[], value: Record<string, any>) {
|
||||
this.set(path, value);
|
||||
}
|
||||
|
||||
private insert_blocks(
|
||||
parentNode: AbstractBlock<B, C> | undefined,
|
||||
blocks: AbstractBlock<B, C>[],
|
||||
placement: 'before' | 'after',
|
||||
referenceNode?: AbstractBlock<B, C>
|
||||
) {
|
||||
if (!blocks || blocks.length === 0 || !parentNode) {
|
||||
return;
|
||||
}
|
||||
// TODO: array equal
|
||||
if (
|
||||
!referenceNode &&
|
||||
parentNode.children.join('') ===
|
||||
blocks.map(node => node.id).join('')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
blocks.forEach(block => {
|
||||
if (block.parent_node) {
|
||||
block.remove();
|
||||
}
|
||||
|
||||
const placement_info = {
|
||||
[placement]:
|
||||
referenceNode?.id ||
|
||||
(parentNode.hasChildNodes() &&
|
||||
parentNode.children[
|
||||
placement === 'before'
|
||||
? 0
|
||||
: parentNode.children.length - 1
|
||||
]),
|
||||
};
|
||||
|
||||
parentNode.insertChildren(
|
||||
block,
|
||||
placement_info[placement] ? placement_info : undefined
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
prepend(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this, blocks.reverse(), 'before');
|
||||
}
|
||||
|
||||
append(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this, blocks, 'after');
|
||||
}
|
||||
|
||||
before(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this.parent_node, blocks, 'before', this);
|
||||
}
|
||||
|
||||
after(...blocks: AbstractBlock<B, C>[]) {
|
||||
this.insert_blocks(this.parent_node, blocks.reverse(), 'after', this);
|
||||
}
|
||||
|
||||
hasChildNodes() {
|
||||
return this.children.length > 0;
|
||||
}
|
||||
|
||||
hasParent(blockId?: string) {
|
||||
let parent = this.parent_node;
|
||||
while (parent) {
|
||||
if (parent.id === blockId) {
|
||||
return true;
|
||||
}
|
||||
parent = parent.parent_node;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: scoped history
|
||||
*/
|
||||
public get history(): HistoryManager {
|
||||
return this.#history;
|
||||
}
|
||||
}
|
||||
321
libs/datasource/jwt/src/block/base.ts
Normal file
321
libs/datasource/jwt/src/block/base.ts
Normal file
@@ -0,0 +1,321 @@
|
||||
import {
|
||||
ArrayOperation,
|
||||
BlockInstance,
|
||||
ContentOperation,
|
||||
ContentTypes,
|
||||
InternalPlainObject,
|
||||
MapOperation,
|
||||
} from '../adapter';
|
||||
import { BlockItem } from '../types';
|
||||
import { getLogger } from '../utils';
|
||||
|
||||
import { AbstractBlock } from './abstract';
|
||||
import { BlockCapability } from './capability';
|
||||
|
||||
const logger = getLogger('BlockDB:block');
|
||||
|
||||
// TODO
|
||||
export interface Decoration extends InternalPlainObject {
|
||||
key: string;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
type Validator = <T>(value: T | undefined) => boolean | void;
|
||||
|
||||
export type IndexMetadata = Readonly<{
|
||||
content?: string;
|
||||
reference?: string;
|
||||
tags: string[];
|
||||
}>;
|
||||
export type QueryMetadata = Readonly<
|
||||
{
|
||||
[key: string]: number | string | string[] | undefined;
|
||||
} & Omit<BlockItem<any>, 'content'>
|
||||
>;
|
||||
|
||||
export type ReadableContentExporter<
|
||||
R = string,
|
||||
T extends ContentTypes = ContentOperation
|
||||
> = (content: MapOperation<T>) => R;
|
||||
type GetExporter<R> = (
|
||||
block: BlockInstance<any>
|
||||
) => Readonly<[string, ReadableContentExporter<R, any>]>[];
|
||||
|
||||
type Exporters = {
|
||||
content: GetExporter<string>;
|
||||
metadata: GetExporter<Array<[string, number | string | string[]]>>;
|
||||
tag: GetExporter<string[]>;
|
||||
};
|
||||
|
||||
export class BaseBlock<
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> extends AbstractBlock<B, C> {
|
||||
readonly #exporters?: Exporters;
|
||||
readonly #content_exporters_getter: () => Map<
|
||||
string,
|
||||
ReadableContentExporter<string, any>
|
||||
>;
|
||||
readonly #metadata_exporters_getter: () => Map<
|
||||
string,
|
||||
ReadableContentExporter<
|
||||
Array<[string, number | string | string[]]>,
|
||||
any
|
||||
>
|
||||
>;
|
||||
readonly #tag_exporters_getter: () => Map<
|
||||
string,
|
||||
ReadableContentExporter<string[], any>
|
||||
>;
|
||||
|
||||
#validators: Map<string, Validator> = new Map();
|
||||
|
||||
constructor(
|
||||
block: B,
|
||||
root?: AbstractBlock<B, C>,
|
||||
parent?: AbstractBlock<B, C>,
|
||||
exporters?: Exporters
|
||||
) {
|
||||
super(block, root, parent);
|
||||
|
||||
this.#exporters = exporters;
|
||||
this.#content_exporters_getter = () =>
|
||||
new Map(exporters?.content(block));
|
||||
this.#metadata_exporters_getter = () =>
|
||||
new Map(exporters?.metadata(block));
|
||||
this.#tag_exporters_getter = () => new Map(exporters?.tag(block));
|
||||
}
|
||||
|
||||
get parent() {
|
||||
return this.parent_node as BaseBlock<B, C> | undefined;
|
||||
}
|
||||
|
||||
private get decoration(): ArrayOperation<Decoration> | undefined {
|
||||
const content = this.getContent<ArrayOperation<Decoration>>();
|
||||
if (!content.has('decoration')) {
|
||||
const decoration = content.createArray<Decoration>();
|
||||
content.set('decoration', decoration);
|
||||
}
|
||||
return content.get('decoration')?.asArray();
|
||||
}
|
||||
|
||||
getDecoration<T = unknown>(key: string): T | undefined {
|
||||
const decoration = this.decoration?.find<Decoration>(
|
||||
decoration => decoration.key === key
|
||||
);
|
||||
if (this.validate(key, decoration?.value)) {
|
||||
return decoration?.value as T;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
getDecorations<T = Record<string, unknown>>(): T {
|
||||
const decorations = {} as T;
|
||||
this.decoration?.forEach(decoration => {
|
||||
const value = this.validate(decoration.key, decoration.value)
|
||||
? decoration.value
|
||||
: undefined;
|
||||
// @ts-ignore
|
||||
decorations[decoration.key] = value;
|
||||
});
|
||||
return decorations;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
getCapability<T extends BlockCapability>(key: string): T | undefined {
|
||||
// TODO: Capability api design
|
||||
return undefined;
|
||||
}
|
||||
|
||||
setDecoration(key: string, value: unknown) {
|
||||
if (!this.validate(key, value)) {
|
||||
throw new Error(`set [${key}] error: validate error.`);
|
||||
}
|
||||
const decoration = { key, value };
|
||||
const index =
|
||||
this.decoration?.findIndex(decoration => decoration.key === key) ??
|
||||
-1;
|
||||
if (index > -1) {
|
||||
this.decoration?.delete(index, 1);
|
||||
this.decoration?.insert(index, [decoration]);
|
||||
} else {
|
||||
this.decoration?.insert(this.decoration?.length, [decoration]);
|
||||
}
|
||||
}
|
||||
|
||||
removeDecoration(key: string) {
|
||||
const index =
|
||||
this.decoration?.findIndex(decoration => decoration.key === key) ??
|
||||
-1;
|
||||
if (index > -1) {
|
||||
this.decoration?.delete(index, 1);
|
||||
}
|
||||
}
|
||||
|
||||
clearDecoration() {
|
||||
this.decoration?.delete(0, this.decoration?.length);
|
||||
}
|
||||
|
||||
setValidator(key: string, validator?: Validator) {
|
||||
if (validator) {
|
||||
this.#validators.set(key, validator);
|
||||
} else {
|
||||
this.#validators.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
private validate(key: string, value: unknown): boolean {
|
||||
const validate = this.#validators.get(key);
|
||||
if (validate) {
|
||||
return validate(value) === false ? false : true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
get group(): BaseBlock<B, C> | undefined {
|
||||
if (this.flavor === 'group') {
|
||||
return this;
|
||||
}
|
||||
return this.parent?.group;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an instance of the child Block
|
||||
* @param blockId block id
|
||||
*/
|
||||
private get_children_instance(blockId?: string): BaseBlock<B, C>[] {
|
||||
return this.get_children(blockId).map(
|
||||
block => new BaseBlock(block, this.root, this, this.#exporters)
|
||||
);
|
||||
}
|
||||
|
||||
private get_indexable_metadata() {
|
||||
const metadata: Record<string, number | string | string[]> = {};
|
||||
for (const [name, exporter] of this.#metadata_exporters_getter()) {
|
||||
try {
|
||||
for (const [key, val] of exporter(this.getContent())) {
|
||||
metadata[key] = val;
|
||||
}
|
||||
} catch (err) {
|
||||
logger(`Failed to export metadata: ${name}`, err);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const parent_page = this._getParentPage(false);
|
||||
if (parent_page) metadata['page'] = parent_page;
|
||||
if (this.group) metadata['group'] = this.group.id;
|
||||
if (this.parent) metadata['parent'] = this.parent.id;
|
||||
} catch (e) {
|
||||
logger(`Failed to export default metadata`, e);
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
public getQueryMetadata(): QueryMetadata {
|
||||
return {
|
||||
type: this.type,
|
||||
flavor: this.flavor,
|
||||
creator: this.creator,
|
||||
children: this.children,
|
||||
created: this.created,
|
||||
updated: this.lastUpdated,
|
||||
...this.get_indexable_metadata(),
|
||||
};
|
||||
}
|
||||
|
||||
private get_indexable_content(): string | undefined {
|
||||
const contents = [];
|
||||
for (const [name, exporter] of this.#content_exporters_getter()) {
|
||||
try {
|
||||
const content = exporter(this.getContent());
|
||||
if (content) contents.push(content);
|
||||
} catch (err) {
|
||||
logger(`Failed to export content: ${name}`, err);
|
||||
}
|
||||
}
|
||||
if (!contents.length) {
|
||||
try {
|
||||
const content = this.getContent() as any;
|
||||
return JSON.stringify(content['toJSON']());
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (e) {}
|
||||
}
|
||||
return contents.join('\n');
|
||||
}
|
||||
|
||||
private get_indexable_tags(): string[] {
|
||||
const tags: string[] = [];
|
||||
for (const [name, exporter] of this.#tag_exporters_getter()) {
|
||||
try {
|
||||
tags.push(...exporter(this.getContent()));
|
||||
} catch (err) {
|
||||
logger(`Failed to export tags: ${name}`, err);
|
||||
}
|
||||
}
|
||||
|
||||
return tags;
|
||||
}
|
||||
|
||||
public getIndexMetadata(): IndexMetadata {
|
||||
return {
|
||||
content: this.get_indexable_content(),
|
||||
reference: '', // TODO: bibliography
|
||||
tags: [...this.getTags(), ...this.get_indexable_tags()],
|
||||
};
|
||||
}
|
||||
|
||||
// ======================================
|
||||
// DOM like apis
|
||||
// ======================================
|
||||
|
||||
get firstChild() {
|
||||
return this.get_children_instance(this.children[0]);
|
||||
}
|
||||
|
||||
get lastChild() {
|
||||
const children = this.children;
|
||||
return this.get_children_instance(children[children.length - 1]);
|
||||
}
|
||||
|
||||
get nextSibling(): BaseBlock<B, C> | undefined {
|
||||
if (this.parent) {
|
||||
const parent = this.parent;
|
||||
const children = parent.children;
|
||||
const index = children.indexOf(this.id);
|
||||
return parent.get_children_instance(children[index + 1])[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
get nextSiblings(): BaseBlock<B, C>[] {
|
||||
if (this.parent) {
|
||||
const parent = this.parent;
|
||||
const children = parent.children;
|
||||
const index = children.indexOf(this.id);
|
||||
return (
|
||||
children
|
||||
.slice(index + 1)
|
||||
.flatMap(id => parent.get_children_instance(id)) || []
|
||||
);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
get previousSibling(): BaseBlock<B, C> | undefined {
|
||||
if (this.parent) {
|
||||
const parent = this.parent;
|
||||
const children = parent.children;
|
||||
const index = children.indexOf(this.id);
|
||||
return parent.get_children_instance(children[index - 1])[0];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
contains(block: AbstractBlock<B, C>) {
|
||||
if (this === block) {
|
||||
return true;
|
||||
}
|
||||
return this.hasChildren(block.id);
|
||||
}
|
||||
}
|
||||
15
libs/datasource/jwt/src/block/capability.ts
Normal file
15
libs/datasource/jwt/src/block/capability.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { BlockSearchItem } from './index';
|
||||
|
||||
export class BlockCapability {
|
||||
// Accept a block instance, check its type, content data structure
|
||||
// Does it meet the structural requirements of the current capability
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
protected check_block(block: BlockSearchItem): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
// data structure upgrade
|
||||
protected migration(): void {
|
||||
// TODO: need to override
|
||||
}
|
||||
}
|
||||
12
libs/datasource/jwt/src/block/index.ts
Normal file
12
libs/datasource/jwt/src/block/index.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import type { BlockMetadata } from './indexer';
|
||||
|
||||
export type BlockSearchItem = Partial<
|
||||
BlockMetadata & {
|
||||
readonly content: string;
|
||||
}
|
||||
>;
|
||||
|
||||
export { BaseBlock } from './base';
|
||||
export type { Decoration, ReadableContentExporter } from './base';
|
||||
export { BlockIndexer } from './indexer';
|
||||
export type { BlockCapability } from './capability';
|
||||
316
libs/datasource/jwt/src/block/indexer.ts
Normal file
316
libs/datasource/jwt/src/block/indexer.ts
Normal file
@@ -0,0 +1,316 @@
|
||||
import { deflateSync, inflateSync, strToU8, strFromU8 } from 'fflate';
|
||||
import { Document as DocumentIndexer, DocumentSearchOptions } from 'flexsearch';
|
||||
import { get, set, keys, del, createStore } from 'idb-keyval';
|
||||
import produce from 'immer';
|
||||
import LRUCache from 'lru-cache';
|
||||
import sift, { Query } from 'sift';
|
||||
|
||||
import {
|
||||
AsyncDatabaseAdapter,
|
||||
BlockInstance,
|
||||
ChangedStates,
|
||||
ContentOperation,
|
||||
} from '../adapter';
|
||||
import { BlockFlavors } from '../types';
|
||||
import { BlockEventBus, getLogger } from '../utils';
|
||||
|
||||
import { BaseBlock, IndexMetadata, QueryMetadata } from './base';
|
||||
|
||||
declare const JWT_DEV: boolean;
|
||||
|
||||
const logger = getLogger('BlockDB:indexing');
|
||||
const logger_debug = getLogger('debug:BlockDB:indexing');
|
||||
|
||||
type ChangedState = ChangedStates extends Map<unknown, infer R> ? R : never;
|
||||
|
||||
export type BlockMetadata = QueryMetadata & { readonly id: string };
|
||||
|
||||
function tokenizeZh(text: string) {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const tokenizer = Intl?.['v8BreakIterator'];
|
||||
if (tokenizer) {
|
||||
const it = tokenizer(['zh-CN'], { type: 'word' });
|
||||
it.adoptText(text);
|
||||
const words = [];
|
||||
|
||||
let cur = 0,
|
||||
prev = 0;
|
||||
|
||||
while (cur < text.length) {
|
||||
prev = cur;
|
||||
cur = it.next();
|
||||
words.push(text.substring(prev, cur));
|
||||
}
|
||||
|
||||
return words;
|
||||
}
|
||||
// eslint-disable-next-line no-control-regex
|
||||
return text.replace(/[\x00-\x7F]/g, '').split('');
|
||||
}
|
||||
|
||||
type IdbInstance = {
|
||||
get: (key: string) => Promise<ArrayBufferLike | undefined>;
|
||||
set: (key: string, value: ArrayBufferLike) => Promise<void>;
|
||||
keys: () => Promise<string[]>;
|
||||
delete: (key: string) => Promise<void>;
|
||||
};
|
||||
|
||||
type BlockIdbInstance = {
|
||||
index: IdbInstance;
|
||||
metadata: IdbInstance;
|
||||
};
|
||||
|
||||
function initIndexIdb(workspace: string): BlockIdbInstance {
|
||||
const index = createStore(`${workspace}_index`, 'index');
|
||||
const metadata = createStore(`${workspace}_metadata`, 'metadata');
|
||||
return {
|
||||
index: {
|
||||
get: (key: string) => get<ArrayBufferLike>(key, index),
|
||||
set: (key: string, value: ArrayBufferLike) =>
|
||||
set(key, value, index),
|
||||
keys: () => keys(index),
|
||||
delete: (key: string) => del(key, index),
|
||||
},
|
||||
metadata: {
|
||||
get: (key: string) => get<ArrayBufferLike>(key, metadata),
|
||||
set: (key: string, value: ArrayBufferLike) =>
|
||||
set(key, value, metadata),
|
||||
keys: () => keys(metadata),
|
||||
delete: (key: string) => del(key, metadata),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type BlockIndexedContent = {
|
||||
index: IndexMetadata;
|
||||
query: QueryMetadata;
|
||||
};
|
||||
|
||||
export type QueryIndexMetadata = Query<QueryMetadata>;
|
||||
|
||||
export class BlockIndexer<
|
||||
A extends AsyncDatabaseAdapter<C>,
|
||||
B extends BlockInstance<C>,
|
||||
C extends ContentOperation
|
||||
> {
|
||||
readonly #adapter: A;
|
||||
readonly #idb: BlockIdbInstance;
|
||||
|
||||
readonly #block_indexer: DocumentIndexer<IndexMetadata>;
|
||||
readonly #block_metadata: LRUCache<string, QueryMetadata>;
|
||||
readonly #event_bus: BlockEventBus;
|
||||
|
||||
readonly #block_builder: (
|
||||
block: BlockInstance<C>
|
||||
) => Promise<BaseBlock<B, C>>;
|
||||
|
||||
readonly #delay_index: { documents: Map<string, BaseBlock<B, C>> };
|
||||
|
||||
constructor(
|
||||
adapter: A,
|
||||
workspace: string,
|
||||
block_builder: (block: BlockInstance<C>) => Promise<BaseBlock<B, C>>,
|
||||
event_bus: BlockEventBus
|
||||
) {
|
||||
this.#adapter = adapter;
|
||||
this.#idb = initIndexIdb(workspace);
|
||||
|
||||
this.#block_indexer = new DocumentIndexer({
|
||||
document: {
|
||||
id: 'id',
|
||||
index: ['content', 'reference'],
|
||||
tag: 'tags',
|
||||
},
|
||||
encode: tokenizeZh,
|
||||
tokenize: 'forward',
|
||||
context: true,
|
||||
});
|
||||
this.#block_metadata = new LRUCache({
|
||||
max: 10240,
|
||||
ttl: 1000 * 60 * 30,
|
||||
});
|
||||
|
||||
this.#block_builder = block_builder;
|
||||
this.#event_bus = event_bus;
|
||||
|
||||
this.#delay_index = { documents: new Map() };
|
||||
|
||||
this.#event_bus
|
||||
.topic('reindex')
|
||||
.on('reindex', this.content_reindex.bind(this), {
|
||||
debounce: { wait: 1000, maxWait: 1000 * 10 },
|
||||
});
|
||||
|
||||
this.#event_bus
|
||||
.topic('save_index')
|
||||
.on('save_index', this.save_index.bind(this), {
|
||||
debounce: { wait: 1000 * 10, maxWait: 1000 * 20 },
|
||||
});
|
||||
}
|
||||
|
||||
private async content_reindex() {
|
||||
const paddings: Record<string, BlockIndexedContent> = {};
|
||||
|
||||
this.#delay_index.documents = produce(
|
||||
this.#delay_index.documents,
|
||||
draft => {
|
||||
for (const [k, block] of draft) {
|
||||
paddings[k] = {
|
||||
index: block.getIndexMetadata(),
|
||||
query: block.getQueryMetadata(),
|
||||
};
|
||||
draft.delete(k);
|
||||
}
|
||||
}
|
||||
);
|
||||
for (const [key, { index, query }] of Object.entries(paddings)) {
|
||||
if (index.content) {
|
||||
await this.#block_indexer.addAsync(key, index);
|
||||
this.#block_metadata.set(key, query);
|
||||
}
|
||||
}
|
||||
this.#event_bus.topic('save_index').emit();
|
||||
}
|
||||
|
||||
private async refresh_index(block: BaseBlock<B, C>) {
|
||||
const filter: string[] = [
|
||||
BlockFlavors.page,
|
||||
BlockFlavors.title,
|
||||
BlockFlavors.heading1,
|
||||
BlockFlavors.heading2,
|
||||
BlockFlavors.heading3,
|
||||
BlockFlavors.text,
|
||||
BlockFlavors.todo,
|
||||
BlockFlavors.reference,
|
||||
];
|
||||
if (filter.includes(block.flavor)) {
|
||||
this.#delay_index.documents = produce(
|
||||
this.#delay_index.documents,
|
||||
draft => {
|
||||
draft.set(block.id, block);
|
||||
}
|
||||
);
|
||||
|
||||
this.#event_bus.topic('reindex').emit();
|
||||
return true;
|
||||
}
|
||||
logger_debug(`skip index ${block.flavor}: ${block.id}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
async refreshIndex(id: string, state: ChangedState) {
|
||||
JWT_DEV && logger(`refreshArticleIndex: ${id}`);
|
||||
if (state === 'delete') {
|
||||
this.#delay_index.documents = produce(
|
||||
this.#delay_index.documents,
|
||||
draft => {
|
||||
this.#block_indexer.remove(id);
|
||||
this.#block_metadata.delete(id);
|
||||
draft.delete(id);
|
||||
}
|
||||
);
|
||||
return;
|
||||
}
|
||||
const block = await this.#adapter.getBlock(id);
|
||||
if (block?.id === id) {
|
||||
if (await this.refresh_index(await this.#block_builder(block))) {
|
||||
JWT_DEV &&
|
||||
logger(
|
||||
state
|
||||
? `refresh index: ${id}, ${state}`
|
||||
: `indexing: ${id}`
|
||||
);
|
||||
} else {
|
||||
JWT_DEV && logger(`skip index: ${id}, ${block.flavor}`);
|
||||
}
|
||||
} else {
|
||||
JWT_DEV && logger(`refreshArticleIndex: ${id} not exists`);
|
||||
}
|
||||
}
|
||||
|
||||
async loadIndex() {
|
||||
for (const key of await this.#idb.index.keys()) {
|
||||
const content = await this.#idb.index.get(key);
|
||||
if (content) {
|
||||
const decoded = strFromU8(inflateSync(new Uint8Array(content)));
|
||||
try {
|
||||
await this.#block_indexer.import(key, decoded as any);
|
||||
} catch (e) {
|
||||
console.error(`Failed to load index ${key}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of await this.#idb.metadata.keys()) {
|
||||
const content = await this.#idb.metadata.get(key);
|
||||
if (content) {
|
||||
const decoded = strFromU8(inflateSync(new Uint8Array(content)));
|
||||
try {
|
||||
await this.#block_indexer.import(key, JSON.parse(decoded));
|
||||
} catch (e) {
|
||||
console.error(`Failed to load index ${key}`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(this.#block_metadata.keys());
|
||||
}
|
||||
|
||||
private async save_index() {
|
||||
const idb = this.#idb;
|
||||
await idb.index
|
||||
.keys()
|
||||
.then(keys => Promise.all(keys.map(key => idb.index.delete(key))));
|
||||
await this.#block_indexer.export((key, data) => {
|
||||
return idb.index.set(
|
||||
String(key),
|
||||
deflateSync(strToU8(data as any))
|
||||
);
|
||||
});
|
||||
const metadata = this.#block_metadata;
|
||||
await idb.metadata
|
||||
.keys()
|
||||
.then(keys =>
|
||||
Promise.all(
|
||||
keys
|
||||
.filter(key => !metadata.has(key))
|
||||
.map(key => idb.metadata.delete(key))
|
||||
)
|
||||
);
|
||||
for (const [key, data] of metadata.entries()) {
|
||||
await idb.metadata.set(
|
||||
key,
|
||||
deflateSync(strToU8(JSON.stringify(data)))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async inspectIndex() {
|
||||
const index: Record<string | number, any> = {};
|
||||
await this.#block_indexer.export((key, data) => {
|
||||
index[key] = data;
|
||||
});
|
||||
}
|
||||
|
||||
public search(
|
||||
part_of_title_or_content:
|
||||
| string
|
||||
| Partial<DocumentSearchOptions<boolean>>
|
||||
) {
|
||||
return this.#block_indexer.search(part_of_title_or_content as string);
|
||||
}
|
||||
|
||||
public query(query: QueryIndexMetadata) {
|
||||
const matches: string[] = [];
|
||||
const filter = sift<QueryMetadata>(query);
|
||||
this.#block_metadata.forEach((value, key) => {
|
||||
if (filter(value)) matches.push(key);
|
||||
});
|
||||
return matches;
|
||||
}
|
||||
|
||||
public getMetadata(ids: string[]): Array<BlockMetadata> {
|
||||
return ids
|
||||
.filter(id => this.#block_metadata.has(id))
|
||||
.map(id => ({ ...this.#block_metadata.get(id)!, id }));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user