/* eslint-disable max-lines */ import { DocumentSearchOptions } from 'flexsearch'; import LRUCache from 'lru-cache'; import { AsyncDatabaseAdapter, YjsAdapter, YjsInitOptions, YjsContentOperation, ChangedStates, BlockListener, BlockInstance, ContentOperation, HistoryManager, ContentTypes, } from './adapter'; import { YjsBlockInstance } from './adapter/yjs'; import { BaseBlock, BlockIndexer, BlockSearchItem, ReadableContentExporter, } from './block'; import { QueryIndexMetadata } from './block/indexer'; import { BlockTypes, BlockTypeKeys, BlockFlavors, BucketBackend, UUID, BlockFlavorKeys, BlockItem, ExcludeFunction, } from './types'; import { BlockEventBus, genUUID, getLogger } from './utils'; declare const JWT_DEV: boolean; const logger = getLogger('BlockDB:client'); // const logger_debug = getLogger('debug:BlockDB:client'); const namedUuid = Symbol('namedUUID'); type BlockUuid = T extends UUID ? T : never; type BlockUuidOrType = T extends | BlockTypeKeys | BlockFlavorKeys ? T : T extends string ? BlockUuid : never; type BlockInstanceValue = ExcludeFunction>; export type BlockMatcher = Partial; type BlockExporters = Map< string, [BlockMatcher, ReadableContentExporter] >; type BlockClientOptions = { content?: BlockExporters; metadata?: BlockExporters>; tagger?: BlockExporters; }; export class BlockClient< A extends AsyncDatabaseAdapter, B extends BlockInstance, C extends ContentOperation > { readonly #adapter: A; readonly #workspace: string; // Maximum cache Block 8192, ttl 30 minutes readonly #block_caches: LRUCache>; readonly #block_indexer: BlockIndexer; readonly #exporters: { readonly content: BlockExporters; readonly metadata: BlockExporters< Array<[string, number | string | string[]]> >; readonly tag: BlockExporters; }; readonly #event_bus: BlockEventBus; readonly #parent_mapping: Map; readonly #page_mapping: Map; readonly #root: { node?: BaseBlock }; private constructor( adapter: A, workspace: string, options?: BlockClientOptions ) { this.#adapter = adapter; this.#workspace = workspace; this.#block_caches = new LRUCache({ max: 8192, ttl: 1000 * 60 * 30 }); this.#exporters = { content: options?.content || new Map(), metadata: options?.metadata || new Map(), tag: options?.tagger || new Map(), }; this.#event_bus = new BlockEventBus(); this.#block_indexer = new BlockIndexer( this.#adapter, this.#workspace, this.block_builder.bind(this), this.#event_bus.topic('indexer') ); this.#parent_mapping = new Map(); this.#page_mapping = new Map(); this.#adapter.on('editing', (states: ChangedStates) => this.#event_bus.topic('editing').emit(states) ); this.#adapter.on('updated', (states: ChangedStates) => this.#event_bus.topic('updated').emit(states) ); this.#event_bus .topic('rebuild_index') .on('rebuild_index', this.rebuild_index.bind(this), { debounce: { wait: 1000, maxWait: 1000 }, }); this.#root = {}; } public addBlockListener(tag: string, listener: BlockListener) { const bus = this.#event_bus.topic('updated'); if (tag !== 'index' || !bus.has(tag)) bus.on(tag, listener); else console.error(`block listener ${tag} is reserved`); } public removeBlockListener(tag: string) { this.#event_bus.topic('updated').off(tag); } public addEditingListener( tag: string, listener: BlockListener> ) { const bus = this.#event_bus.topic>>('editing'); if (tag !== 'index' || !bus.has(tag)) bus.on(tag, listener); else console.error(`editing listener ${tag} is reserved`); } public removeEditingListener(tag: string) { this.#event_bus.topic('editing').off(tag); } private inspector() { return { ...this.#adapter.inspector(), indexed: () => this.#block_indexer.inspectIndex(), }; } private async rebuild_index(exists_ids?: string[]) { JWT_DEV && logger(`rebuild index`); const blocks = await this.#adapter.getBlockByType(BlockTypes.block); const excluded = exists_ids || []; await Promise.all( blocks .filter(id => !excluded.includes(id)) .map(id => this.#block_indexer.refreshIndex(id, 'add')) ); } public async buildIndex() { JWT_DEV && logger(`buildIndex: start`); // Skip the block index that exists in the metadata, assuming that the index of the block existing in the metadata is the latest, and modify this part if there is a problem // Although there may be cases where the index is refreshed but the metadata is not refreshed, re-indexing will be automatically triggered after the block is changed const exists_ids = await this.#block_indexer.loadIndex(); await this.rebuild_index(exists_ids); this.addBlockListener('index', async states => { await Promise.allSettled( Array.from(states.entries()).map(([id, state]) => { if (state === 'delete') this.#block_caches.delete(id); return this.#block_indexer.refreshIndex(id, state); }) ); }); } /** * Get a specific type of block, currently only the article type is supported * @param block_type block type * @returns */ public async getByType( block_type: BlockTypeKeys | BlockFlavorKeys ): Promise>> { JWT_DEV && logger(`getByType: ${block_type}`); const ids = [ ...this.#block_indexer.query({ type: BlockTypes[block_type as BlockTypeKeys], }), ...this.#block_indexer.query({ flavor: BlockFlavors[block_type as BlockFlavorKeys], }), ]; const docs = await Promise.all( ids.map(id => this.get(id as BlockUuidOrType).then( doc => [id, doc] as const ) ) ); return new Map(docs.filter(([, doc]) => doc.children.length)); } /** * research all * @param part_of_title_or_content Title or content keyword, support Chinese * @param part_of_title_or_content.index search range, optional values: title, ttl, content, reference * @param part_of_title_or_content.tag tag, string or array of strings, supports multiple tags * @param part_of_title_or_content.query keyword, support Chinese * @param part_of_title_or_content.limit The limit of the number of search results, the default is 100 * @param part_of_title_or_content.offset search result offset, used for page turning, default is 0 * @param part_of_title_or_content.suggest Fuzzy matching, after enabling the content including some keywords can also be searched, the default is false * @returns array of search results, each array is a list of attributed block ids */ public search( part_of_title_or_content: | string | Partial> ) { return this.#block_indexer.search(part_of_title_or_content); } /** * Full text search, the returned results are grouped by page dimension * @param part_of_title_or_content Title or content keyword, support Chinese * @param part_of_title_or_content.index search range, optional values: title, ttl, content, reference * @param part_of_title_or_content.tag tag, string or array of strings, supports multiple tags * @param part_of_title_or_content.query keyword, support Chinese * @param part_of_title_or_content.limit The limit of the number of search results, the default is 100 * @param part_of_title_or_content.offset search result offset, used for page turning, default is 0 * @param part_of_title_or_content.suggest Fuzzy matching, after enabling the content including some keywords can also be searched, the default is false * @returns array of search results, each array is a page */ public async searchPages( part_of_title_or_content: | string | Partial> ): Promise { const promised_pages = await Promise.all( this.search(part_of_title_or_content).flatMap(({ result }) => result.map(async id => { const page = this.#page_mapping.get(id as string); if (page) return page; const block = await this.get(id as BlockTypeKeys); return this.set_page(block); }) ) ); const pages = [ ...new Set(promised_pages.filter((v): v is string => !!v)), ]; return Promise.all( this.#block_indexer.getMetadata(pages).map(async page => ({ content: this.get_decoded_content( await this.#adapter.getBlock(page.id) ), ...page, })) ); } /** * Inquire * @returns array of search results */ public query(query: QueryIndexMetadata): string[] { return this.#block_indexer.query(query); } /** * Get a fixed name, which has the same UUID in each workspace, and is automatically created when it does not exist * Generally used to store workspace-level global configuration * @param name block name * @returns block instance */ private async get_named_block( name: string, options?: { workspace?: boolean } ): Promise> { const block = await this.get(genUUID(name), { flavor: options?.workspace ? BlockFlavors.workspace : BlockFlavors.page, [namedUuid]: true, }); return block; } /** * Get the workspace block of the current instance * @returns block instance */ public async getWorkspace() { if (!this.#root.node) { this.#root.node = await this.get_named_block(this.#workspace, { workspace: true, }); } return this.#root.node; } /** * @deprecated custom data including access to ws configuration is unified through baseBlock.get/setDecoration(key). * - Get the config of the workspace block of the current instance * @returns MapOperation */ public async getWorkspaceConfig() { return (await this.getWorkspace()).getContent(); } private async get_parent(id: string) { const parents = this.#parent_mapping.get(id); if (parents) { const parent_block_id = parents[0]; if (!this.#block_caches.has(parent_block_id)) { this.#block_caches.set( parent_block_id, await this.get(parent_block_id as BlockTypeKeys) ); } return this.#block_caches.get(parent_block_id); } return undefined; } private set_parent(parent: string, child: string) { const parents = this.#parent_mapping.get(child); if (parents?.length) { if (!parents.includes(parent)) { console.error('parent already exists', child, parents); this.#parent_mapping.set(child, [...parents, parent]); } } else { this.#parent_mapping.set(child, [parent]); } } private set_page(block: BaseBlock) { const page = this.#page_mapping.get(block.id); if (page) return page; const parent_page = block.parent_page; if (parent_page) { this.#page_mapping.set(block.id, parent_page); return parent_page; } return undefined; } registerContentExporter( name: string, matcher: BlockMatcher, exporter: ReadableContentExporter ) { this.#exporters.content.set(name, [matcher, exporter]); this.#event_bus.topic('rebuild_index').emit(); // // rebuild the index every time the content exporter is registered } unregisterContentExporter(name: string) { this.#exporters.content.delete(name); this.#event_bus.topic('rebuild_index').emit(); // Rebuild indexes every time content exporter logs out } registerMetadataExporter( name: string, matcher: BlockMatcher, exporter: ReadableContentExporter< Array<[string, number | string | string[]]>, T > ) { this.#exporters.metadata.set(name, [matcher, exporter]); this.#event_bus.topic('rebuild_index').emit(); // // rebuild the index every time the content exporter is registered } unregisterMetadataExporter(name: string) { this.#exporters.metadata.delete(name); this.#event_bus.topic('rebuild_index').emit(); // Rebuild indexes every time content exporter logs out } registerTagExporter( name: string, matcher: BlockMatcher, exporter: ReadableContentExporter ) { this.#exporters.tag.set(name, [matcher, exporter]); this.#event_bus.topic('rebuild_index').emit(); // Reindex every tag exporter registration } unregisterTagExporter(name: string) { this.#exporters.tag.delete(name); this.#event_bus.topic('rebuild_index').emit(); // Reindex every time tag exporter logs out } private get_exporters( exporter_map: BlockExporters, block: BlockInstance ): Readonly<[string, ReadableContentExporter]>[] { const exporters = []; for (const [name, [cond, exporter]] of exporter_map) { const conditions = Object.entries(cond); let matched = 0; for (const [key, value] of conditions) { if (block[key as keyof BlockInstanceValue] === value) { matched += 1; } } if (matched === conditions.length) exporters.push([name, exporter] as const); } return exporters; } private get_decoded_content(block?: BlockInstance) { if (block) { const [exporter] = this.get_exporters( this.#exporters.content, block ); if (exporter) { const op = block.content.asMap(); if (op) return exporter[1](op); } } return undefined; } private async block_builder( block: BlockInstance, root?: BaseBlock ) { return new BaseBlock( block, root, (await this.get_parent(block.id)) || root, { content: block => this.get_exporters(this.#exporters.content, block), metadata: block => this.get_exporters(this.#exporters.metadata, block), tag: block => this.get_exporters(this.#exporters.tag, block), } ); } /** * Get a Block, which is automatically created if it does not exist * @param block_id_or_type block id, create a new text block when BlockTypes/BlockFlavors are not provided, does not exist or is provided. If BlockTypes/BlockFlavors are provided, create a block of the corresponding type * @param options.type The type of block created when block does not exist, the default is block * @param options.flavor The flavor of the block created when the block does not exist, the default is text * @param options.binary content of binary block, must be provided when type or block_id_or_type is binary * @returns block instance */ public async get( block_id_or_type?: BlockUuidOrType, options?: { type?: BlockItem['type']; flavor: BlockItem['flavor']; binary?: ArrayBuffer; [namedUuid]?: boolean; } ): Promise> { JWT_DEV && logger(`get: ${block_id_or_type}`); const { type = BlockTypes.block, flavor = BlockFlavors.text, binary, [namedUuid]: is_named_uuid, } = options || {}; if (block_id_or_type && this.#block_caches.has(block_id_or_type)) { return this.#block_caches.get(block_id_or_type) as BaseBlock; } else { const block = (block_id_or_type && (await this.#adapter.getBlock(block_id_or_type))) || (await this.#adapter.createBlock({ uuid: is_named_uuid ? block_id_or_type : undefined, binary, type: block_id_or_type && BlockTypes[block_id_or_type as BlockTypeKeys] ? BlockTypes[block_id_or_type as BlockTypeKeys] : type, flavor: block_id_or_type && BlockFlavors[block_id_or_type as BlockFlavorKeys] ? BlockFlavors[block_id_or_type as BlockFlavorKeys] : flavor, })); const root = is_named_uuid ? undefined : await this.getWorkspace(); for (const child of block.children) { this.set_parent(block.id, child); } const abstract_block = await this.block_builder(block, root); this.set_page(abstract_block); abstract_block.on('parent', 'client_hook', state => { const [parent] = state.keys(); this.set_parent(parent, abstract_block.id); this.set_page(abstract_block); }); this.#block_caches.set(abstract_block.id, abstract_block); return abstract_block; } } public async getBlockByFlavor( flavor: BlockItem['flavor'] ): Promise { return await this.#adapter.getBlockByFlavor(flavor); } public getUserId(): string { return this.#adapter.getUserId(); } public has(block_ids: string[]): Promise { return this.#adapter.checkBlocks(block_ids); } /** * Suspend instant update event dispatch, extend to a maximum of 500ms once, and a maximum of 2000ms when triggered continuously * @param suspend true: suspend monitoring, false: resume monitoring */ suspend(suspend: boolean) { this.#adapter.suspend(suspend); } public get history(): HistoryManager { return this.#adapter.history(); } public static async init( workspace: string, options: Partial = {} ): Promise { const instance = await YjsAdapter.init(workspace, { backend: BucketBackend.YjsWebSocketAffine, ...options, }); return new BlockClient(instance, workspace, options); } } export type BlockImplInstance = BaseBlock< YjsBlockInstance, YjsContentOperation >; export type BlockClientInstance = BlockClient< YjsAdapter, YjsBlockInstance, YjsContentOperation >; export type BlockInitOptions = NonNullable< Parameters[1] >; export type { TextOperation, ArrayOperation, MapOperation, ChangedStates, } from './adapter'; export type { BlockSearchItem, Decoration as BlockDecoration, ReadableContentExporter as BlockContentExporter, } from './block'; export type { BlockTypeKeys } from './types'; export { BlockTypes, BucketBackend as BlockBackend } from './types'; export { isBlock } from './utils'; export type { QueryIndexMetadata };