feat(nbstore): add indexer storage (#10953)

packages/common/nbstore/src/storage/doc.ts

@@ -47,6 +47,7 @@ export interface DocStorageOptions {
 export interface DocStorage extends Storage {
   readonly storageType: 'doc';
   readonly isReadonly: boolean;
+  readonly spaceId: string;
   /**
    * Get a doc record with latest binary.
    */
@@ -103,7 +104,7 @@ export abstract class DocStorageBase<Opts = {}> implements DocStorage {
   readonly storageType = 'doc';
   abstract readonly connection: Connection;
   protected readonly locker: Locker = new SingletonLocker();
-  protected readonly spaceId = this.options.id;
+  readonly spaceId = this.options.id;
 
   constructor(protected readonly options: Opts & DocStorageOptions) {}
 

packages/common/nbstore/src/storage/dummy/indexer-sync.ts

@@ -0,0 +1,16 @@
+import { DummyConnection } from '../../connection';
+import type { DocClock } from '../doc';
+import { IndexerSyncStorageBase } from '../indexer-sync';
+
+export class DummyIndexerSyncStorage extends IndexerSyncStorageBase {
+  override connection = new DummyConnection();
+  override getDocIndexedClock(_docId: string): Promise<DocClock | null> {
+    return Promise.resolve(null);
+  }
+  override setDocIndexedClock(_docClock: DocClock): Promise<void> {
+    return Promise.resolve();
+  }
+  override clearDocIndexedClock(_docId: string): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/common/nbstore/src/storage/dummy/indexer.ts

@@ -0,0 +1,88 @@
+import { NEVER, type Observable } from 'rxjs';
+
+import { DummyConnection } from '../../connection';
+import {
+  type AggregateOptions,
+  type AggregateResult,
+  type IndexerDocument,
+  type IndexerSchema,
+  IndexerStorageBase,
+  type Query,
+  type SearchOptions,
+  type SearchResult,
+} from '../indexer';
+
+export class DummyIndexerStorage extends IndexerStorageBase {
+  readonly isReadonly = true;
+  readonly connection = new DummyConnection();
+
+  override search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Promise<SearchResult<T, O>> {
+    return Promise.resolve({
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      nodes: [],
+    });
+  }
+  override aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Promise<AggregateResult<T, O>> {
+    return Promise.resolve({
+      pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+      buckets: [],
+    });
+  }
+  override search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(_table: T, _query: Query<T>, _options?: O): Observable<SearchResult<T, O>> {
+    return NEVER;
+  }
+  override aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    _table: T,
+    _query: Query<T>,
+    _field: keyof IndexerSchema[T],
+    _options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return NEVER;
+  }
+
+  override deleteByQuery<T extends keyof IndexerSchema>(
+    _table: T,
+    _query: Query<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+
+  override insert<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override delete<T extends keyof IndexerSchema>(
+    _table: T,
+    _id: string
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override update<T extends keyof IndexerSchema>(
+    _table: T,
+    _document: IndexerDocument<T>
+  ): Promise<void> {
+    return Promise.resolve();
+  }
+  override refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
+    return Promise.resolve();
+  }
+}

packages/common/nbstore/src/storage/index.ts

@@ -10,6 +10,10 @@ import { DummyBlobStorage } from './dummy/blob';
 import { DummyBlobSyncStorage } from './dummy/blob-sync';
 import { DummyDocStorage } from './dummy/doc';
 import { DummyDocSyncStorage } from './dummy/doc-sync';
+import { DummyIndexerStorage } from './dummy/indexer';
+import { DummyIndexerSyncStorage } from './dummy/indexer-sync';
+import type { IndexerStorage } from './indexer';
+import type { IndexerSyncStorage } from './indexer-sync';
 import type { StorageType } from './storage';
 
 type Storages =
@@ -17,7 +21,9 @@ type Storages =
   | BlobStorage
   | BlobSyncStorage
   | DocSyncStorage
-  | AwarenessStorage;
+  | AwarenessStorage
+  | IndexerStorage
+  | IndexerSyncStorage;
 
 export type SpaceStorageOptions = {
   [K in StorageType]?: Storages & { storageType: K };
@@ -37,6 +43,8 @@ export class SpaceStorage {
       blobSync: storages.blobSync ?? new DummyBlobSyncStorage(),
       doc: storages.doc ?? new DummyDocStorage(),
       docSync: storages.docSync ?? new DummyDocSyncStorage(),
+      indexer: storages.indexer ?? new DummyIndexerStorage(),
+      indexerSync: storages.indexerSync ?? new DummyIndexerSyncStorage(),
     };
   }
 
@@ -83,4 +91,5 @@ export * from './doc';
 export * from './doc-sync';
 export * from './errors';
 export * from './history';
+export * from './indexer';
 export * from './storage';

packages/common/nbstore/src/storage/indexer-sync.ts

@@ -0,0 +1,21 @@
+import type { Connection } from '../connection';
+import type { DocClock } from './doc';
+import type { Storage } from './storage';
+
+export interface IndexerSyncStorage extends Storage {
+  readonly storageType: 'indexerSync';
+
+  getDocIndexedClock(docId: string): Promise<DocClock | null>;
+
+  setDocIndexedClock(docClock: DocClock): Promise<void>;
+
+  clearDocIndexedClock(docId: string): Promise<void>;
+}
+
+export abstract class IndexerSyncStorageBase implements IndexerSyncStorage {
+  readonly storageType = 'indexerSync';
+  abstract connection: Connection<any>;
+  abstract getDocIndexedClock(docId: string): Promise<DocClock | null>;
+  abstract setDocIndexedClock(docClock: DocClock): Promise<void>;
+  abstract clearDocIndexedClock(docId: string): Promise<void>;
+}

packages/common/nbstore/src/storage/indexer.ts

@@ -0,0 +1,176 @@
+export * from './indexer/document';
+export * from './indexer/field-type';
+export * from './indexer/query';
+export * from './indexer/schema';
+
+import type { Observable } from 'rxjs';
+
+import type { Connection } from '../connection';
+import type { IndexerDocument } from './indexer/document';
+import type { Query } from './indexer/query';
+import type { IndexerSchema } from './indexer/schema';
+import type { Storage } from './storage';
+
+export interface IndexerStorage extends Storage {
+  readonly storageType: 'indexer';
+  readonly isReadonly: boolean;
+
+  search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Promise<SearchResult<T, O>>;
+
+  aggregate<T extends keyof IndexerSchema, const O extends AggregateOptions<T>>(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>>;
+
+  search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Observable<SearchResult<T, O>>;
+
+  aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>>;
+
+  deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void>;
+
+  insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  delete<T extends keyof IndexerSchema>(table: T, id: string): Promise<void>;
+
+  update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
+}
+
+type ResultPagination = {
+  count: number;
+  limit: number;
+  skip: number;
+  hasMore: boolean;
+};
+
+type PaginationOption = { limit?: number; skip?: number };
+
+type HighlightAbleField<T extends keyof IndexerSchema> = {
+  [K in keyof IndexerSchema[T]]: IndexerSchema[T][K] extends 'FullText'
+    ? K
+    : never;
+}[keyof IndexerSchema[T]];
+
+export type SearchOptions<T extends keyof IndexerSchema> = {
+  pagination?: PaginationOption;
+  highlights?: { field: HighlightAbleField<T>; before: string; end: string }[];
+  fields?: (keyof IndexerSchema[T])[];
+};
+
+export type SearchResult<
+  T extends keyof IndexerSchema,
+  O extends SearchOptions<T>,
+> = {
+  pagination: ResultPagination;
+  nodes: ({ id: string; score: number } & (O['fields'] extends any[]
+    ? { fields: { [key in O['fields'][number]]: string | string[] } }
+    : unknown) &
+    (O['highlights'] extends any[]
+      ? { highlights: { [key in O['highlights'][number]['field']]: string[] } }
+      : unknown))[];
+};
+
+export interface AggregateOptions<T extends keyof IndexerSchema> {
+  pagination?: PaginationOption;
+  hits?: SearchOptions<T>;
+}
+
+export type AggregateResult<
+  T extends keyof IndexerSchema,
+  O extends AggregateOptions<T>,
+> = {
+  pagination: ResultPagination;
+  buckets: ({
+    key: string;
+    score: number;
+    count: number;
+  } & (O['hits'] extends object
+    ? { hits: SearchResult<T, O['hits']> }
+    : unknown))[];
+};
+
+export abstract class IndexerStorageBase implements IndexerStorage {
+  readonly storageType = 'indexer';
+  abstract readonly connection: Connection;
+  abstract readonly isReadonly: boolean;
+
+  abstract search<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>>;
+
+  abstract aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>>;
+
+  abstract search$<
+    T extends keyof IndexerSchema,
+    const O extends SearchOptions<T>,
+  >(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>>;
+
+  abstract aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>>;
+
+  abstract deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void>;
+
+  abstract insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  abstract delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void>;
+
+  abstract update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void>;
+
+  abstract refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
+}

packages/common/nbstore/src/storage/indexer/document.ts

@@ -0,0 +1,58 @@
+import type { IndexerSchema } from './schema';
+
+export class IndexerDocument<
+  S extends keyof IndexerSchema = keyof IndexerSchema,
+> {
+  constructor(public readonly id: string) {}
+
+  fields = new Map<keyof IndexerSchema[S], string[]>();
+
+  public insert<F extends keyof IndexerSchema[S]>(
+    field: F,
+    value: string | string[]
+  ) {
+    const values = this.fields.get(field) ?? [];
+    if (Array.isArray(value)) {
+      values.push(...value);
+    } else {
+      values.push(value);
+    }
+    this.fields.set(field, values);
+  }
+
+  get<F extends keyof IndexerSchema[S]>(
+    field: F
+  ): string[] | string | undefined {
+    const values = this.fields.get(field);
+    if (values === undefined) {
+      return undefined;
+    } else if (values.length === 1) {
+      return values[0];
+    } else {
+      return values;
+    }
+  }
+
+  static from<S extends keyof IndexerSchema>(
+    id: string,
+    map:
+      | Partial<Record<keyof IndexerSchema[S], string | string[]>>
+      | Map<keyof IndexerSchema[S], string | string[]>
+  ): IndexerDocument<S> {
+    const doc = new IndexerDocument<S>(id);
+
+    if (map instanceof Map) {
+      for (const [key, value] of map) {
+        doc.insert(key, value);
+      }
+    } else {
+      for (const key in map) {
+        if (map[key] === undefined || map[key] === null) {
+          continue;
+        }
+        doc.insert(key, map[key]);
+      }
+    }
+    return doc;
+  }
+}

packages/common/nbstore/src/storage/indexer/field-type.ts

@@ -0,0 +1 @@
+export type IndexFieldType = 'Integer' | 'FullText' | 'String' | 'Boolean';

packages/common/nbstore/src/storage/indexer/query.ts

@@ -0,0 +1,35 @@
+import type { IndexerSchema } from './schema';
+
+export type MatchQuery<T extends keyof IndexerSchema> = {
+  type: 'match';
+  field: keyof IndexerSchema[T];
+  match: string;
+};
+
+export type BoostQuery = {
+  type: 'boost';
+  query: Query<any>;
+  boost: number;
+};
+
+export type BooleanQuery<T extends keyof IndexerSchema> = {
+  type: 'boolean';
+  occur: 'should' | 'must' | 'must_not';
+  queries: Query<T>[];
+};
+
+export type ExistsQuery<T extends keyof IndexerSchema> = {
+  type: 'exists';
+  field: keyof IndexerSchema[T];
+};
+
+export type AllQuery = {
+  type: 'all';
+};
+
+export type Query<T extends keyof IndexerSchema> =
+  | BooleanQuery<T>
+  | MatchQuery<T>
+  | AllQuery
+  | ExistsQuery<T>
+  | BoostQuery;

packages/common/nbstore/src/storage/indexer/schema.ts

@@ -0,0 +1,51 @@
+import type { IndexFieldType } from './field-type';
+
+export const IndexerSchema = {
+  doc: {
+    docId: 'String',
+    title: 'FullText',
+    // summary of the doc, used for preview
+    summary: { type: 'String', index: false },
+  },
+  block: {
+    docId: 'String',
+    blockId: 'String',
+    content: 'FullText',
+    flavour: 'String',
+    blob: 'String',
+    // reference doc id
+    // ['xxx','yyy']
+    refDocId: 'String',
+    // reference info, used for backlink to specific block
+    // [{"docId":"xxx","mode":"page","blockIds":["gt5Yfq1maYvgNgpi13rIq"]},{"docId":"yyy","mode":"edgeless","blockIds":["k5prpOlDF-9CzfatmO0W7"]}]
+    ref: { type: 'String', index: false },
+    // parent block flavour
+    parentFlavour: 'String',
+    // parent block id
+    parentBlockId: 'String',
+    // additional info
+    // { "databaseName": "xxx", "displayMode": "page/edgeless", "noteBlockId": "xxx" }
+    additional: { type: 'String', index: false },
+    markdownPreview: { type: 'String', index: false },
+  },
+} satisfies Record<string, Record<string, IndexerFieldSchema>>;
+
+export type IndexerFieldSchema =
+  | IndexFieldType
+  | {
+      type: IndexFieldType;
+      /**
+       * If false, the field will not be indexed, and thus not searchable.
+       *
+       * default: true
+       */
+      index?: boolean;
+      /**
+       * If false, the field will not be stored, and not included in the search result.
+       *
+       * default: true
+       */
+      store?: boolean;
+    };
+
+export type IndexerSchema = typeof IndexerSchema;

packages/common/nbstore/src/storage/storage.ts

@@ -1,6 +1,13 @@
 import type { Connection } from '../connection';
 
-export type StorageType = 'blob' | 'blobSync' | 'doc' | 'docSync' | 'awareness';
+export type StorageType =
+  | 'blob'
+  | 'blobSync'
+  | 'doc'
+  | 'docSync'
+  | 'awareness'
+  | 'indexer'
+  | 'indexerSync';
 
 export interface Storage {
   readonly storageType: StorageType;