feat: improve indexing perf with native indexer (#14066)

fix #12132, #14006, #13496, #12375, #12132 

The previous idb indexer generated a large number of scattered writes
when flushing to disk, which caused CPU and disk write spikes. If the
document volume is extremely large, the accumulation of write
transactions will cause memory usage to continuously increase.

This PR introduces batch writes to mitigate write performance on the web
side, and adds a native indexer on the Electron side to greatly improve
performance.


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Full-text search (FTS) added across storage layers and native plugins:
indexing, search, document retrieval, match ranges, and index flushing.
* New SQLite-backed indexer storage, streaming search/aggregate APIs,
and in-memory index with node-building and highlighting.

* **Performance**
* Indexing rewritten for batched, concurrent writes and parallel
metadata updates.
* Search scoring enhanced to consider multiple term positions and
aggregated term data.

* **Other**
  * Configurable refresh interval and indexer version bump.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/common/nbstore/src/impls/idb/indexer/data-struct.ts

@@ -180,9 +180,9 @@ export class DataStruct {
       .index('nid')
       .getAllKeys(nid);
 
-    for (const indexId of indexIds) {
-      await trx.objectStore('invertedIndex').delete(indexId);
-    }
+    await Promise.all(
+      indexIds.map(indexId => trx.objectStore('invertedIndex').delete(indexId))
+    );
   }
 
   private async delete(

packages/common/nbstore/src/impls/idb/indexer/index.ts

@@ -18,6 +18,7 @@ import { backoffRetry, exhaustMapWithTrailing } from './utils';
 
 export class IndexedDBIndexerStorage extends IndexerStorageBase {
   static readonly identifier = 'IndexedDBIndexerStorage';
+  override recommendRefreshInterval: number = 0; // force refresh on each indexer operation
   readonly connection = share(new IDBConnection(this.options));
   override isReadonly = false;
   private readonly data = new DataStruct();

packages/common/nbstore/src/impls/idb/indexer/inverted-index.ts

@@ -68,13 +68,16 @@ export class StringInvertedIndex implements InvertedIndex {
   }
 
   async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
-    for (const term of terms) {
-      await trx.objectStore('invertedIndex').put({
-        table: this.table,
-        key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
-        nid: id,
-      });
-    }
+    const uniqueTerms = new Set(terms);
+    await Promise.all(
+      Array.from(uniqueTerms).map(term =>
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
+          nid: id,
+        })
+      )
+    );
   }
 }
 
@@ -127,13 +130,16 @@ export class IntegerInvertedIndex implements InvertedIndex {
   }
 
   async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
-    for (const term of terms) {
-      await trx.objectStore('invertedIndex').put({
-        table: this.table,
-        key: InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
-        nid: id,
-      });
-    }
+    const uniqueTerms = new Set(terms);
+    await Promise.all(
+      Array.from(uniqueTerms).map(term =>
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
+          nid: id,
+        })
+      )
+    );
   }
 }
 
@@ -186,16 +192,19 @@ export class BooleanInvertedIndex implements InvertedIndex {
   }
 
   async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
-    for (const term of terms) {
-      await trx.objectStore('invertedIndex').put({
-        table: this.table,
-        key: InvertedIndexKey.forBoolean(
-          this.fieldKey,
-          term === 'true'
-        ).buffer(),
-        nid: id,
-      });
-    }
+    const uniqueTerms = new Set(terms);
+    await Promise.all(
+      Array.from(uniqueTerms).map(term =>
+        trx.objectStore('invertedIndex').put({
+          table: this.table,
+          key: InvertedIndexKey.forBoolean(
+            this.fieldKey,
+            term === 'true'
+          ).buffer(),
+          nid: id,
+        })
+      )
+    );
   }
 }
 
@@ -260,37 +269,37 @@ export class FullTextInvertedIndex implements InvertedIndex {
         const key = InvertedIndexKey.fromBuffer(obj.key);
         const originTokenTerm = key.asString();
         const matchLength = token.term.length;
-        const position = obj.pos ?? {
-          i: 0,
-          l: 0,
-          rs: [],
-        };
-        const termFreq = position.rs.length;
-        const totalCount = objs.length;
-        const fieldLength = position.l;
-        const score =
-          bm25(termFreq, 1, totalCount, fieldLength, avgFieldLength) *
-          (matchLength / originTokenTerm.length);
-        const match = {
-          score,
-          positions: new Map(),
-        };
-        const ranges = match.positions.get(position.i) || [];
-        ranges.push(
-          ...position.rs.map(([start, _end]) => [start, start + matchLength])
-        );
-        match.positions.set(position.i, ranges);
-        submatched.push({
-          nid: obj.nid,
-          score,
-          position: {
-            index: position.i,
-            ranges: position.rs.map(([start, _end]) => [
-              start,
-              start + matchLength,
-            ]),
-          },
-        });
+        let positions = obj.pos
+          ? Array.isArray(obj.pos)
+            ? obj.pos
+            : [obj.pos]
+          : [
+              {
+                i: 0,
+                l: 0,
+                rs: [] as [number, number][],
+              },
+            ];
+
+        for (const position of positions) {
+          const termFreq = position.rs.length;
+          const totalCount = objs.length;
+          const fieldLength = position.l;
+          const score =
+            bm25(termFreq, 1, totalCount, fieldLength, avgFieldLength) *
+            (matchLength / originTokenTerm.length);
+          submatched.push({
+            nid: obj.nid,
+            score,
+            position: {
+              index: position.i,
+              ranges: position.rs.map(([start, _end]) => [
+                start,
+                start + matchLength,
+              ]),
+            },
+          });
+        }
       }
 
       // normalize score
@@ -369,6 +378,13 @@ export class FullTextInvertedIndex implements InvertedIndex {
   }
 
   async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
+    const promises: Promise<any>[] = [];
+    const totalTermLength = terms.reduce((acc, term) => acc + term.length, 0);
+    const globalTokenMap = new Map<
+      string,
+      { l: number; i: number; rs: [number, number][] }[]
+    >();
+
     for (let i = 0; i < terms.length; i++) {
       const tokenMap = new Map<string, Token[]>();
       const originString = terms[i];
@@ -382,44 +398,57 @@ export class FullTextInvertedIndex implements InvertedIndex {
       }
 
       for (const [term, tokens] of tokenMap) {
-        await trx.objectStore('invertedIndex').put({
+        const entry = globalTokenMap.get(term) || [];
+        entry.push({
+          l: originString.length,
+          i: i,
+          rs: tokens.map(token => [token.start, token.end]),
+        });
+        globalTokenMap.set(term, entry);
+      }
+    }
+
+    for (const [term, positions] of globalTokenMap) {
+      promises.push(
+        trx.objectStore('invertedIndex').put({
           table: this.table,
           key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
           nid: id,
-          pos: {
-            l: originString.length,
-            i: i,
-            rs: tokens.map(token => [token.start, token.end]),
-          },
-        });
-      }
-
-      const indexerMetadataStore = trx.objectStore('indexerMetadata');
-      // update avg-field-length
-      const totalCount =
-        (
-          await indexerMetadataStore.get(
-            `full-text:field-count:${this.table}:${this.fieldKey}`
-          )
-        )?.value ?? 0;
-      const avgFieldLength =
-        (
-          await indexerMetadataStore.get(
-            `full-text:avg-field-length:${this.table}:${this.fieldKey}`
-          )
-        )?.value ?? 0;
-      await indexerMetadataStore.put({
-        key: `full-text:field-count:${this.table}:${this.fieldKey}`,
-        value: totalCount + 1,
-      });
-      await indexerMetadataStore.put({
-        key: `full-text:avg-field-length:${this.table}:${this.fieldKey}`,
-        value:
-          avgFieldLength +
-          (terms.reduce((acc, term) => acc + term.length, 0) - avgFieldLength) /
-            (totalCount + 1),
-      });
+          pos: positions,
+        })
+      );
     }
+
+    const indexerMetadataStore = trx.objectStore('indexerMetadata');
+    const countKey = `full-text:field-count:${this.table}:${this.fieldKey}`;
+    const avgKey = `full-text:avg-field-length:${this.table}:${this.fieldKey}`;
+
+    const [countObj, avgObj] = await Promise.all([
+      indexerMetadataStore.get(countKey),
+      indexerMetadataStore.get(avgKey),
+    ]);
+
+    const totalCount = countObj?.value ?? 0;
+    const avgFieldLength = avgObj?.value ?? 0;
+
+    const newTotalCount = totalCount + terms.length;
+    const newAvgFieldLength =
+      (avgFieldLength * totalCount + totalTermLength) / newTotalCount;
+
+    promises.push(
+      indexerMetadataStore.put({
+        key: countKey,
+        value: newTotalCount,
+      })
+    );
+    promises.push(
+      indexerMetadataStore.put({
+        key: avgKey,
+        value: isNaN(newAvgFieldLength) ? 0 : newAvgFieldLength,
+      })
+    );
+
+    await Promise.all(promises);
   }
 }
 

packages/common/nbstore/src/impls/idb/schema.ts

@@ -162,7 +162,7 @@ export interface DocStorageSchema extends DBSchema {
         i: number /* index */;
         l: number /* length */;
         rs: [number, number][] /* ranges: [start, end] */;
-      };
+      }[];
       key: ArrayBuffer;
     };
     indexes: { key: [string, ArrayBuffer]; nid: number };

packages/common/nbstore/src/impls/sqlite/db.ts

@@ -83,6 +83,35 @@ export interface NativeDBApis {
     blobId: string
   ) => Promise<Date | null>;
   crawlDocData: (id: string, docId: string) => Promise<CrawlResult>;
+  ftsAddDocument: (
+    id: string,
+    indexName: string,
+    docId: string,
+    text: string,
+    index: boolean
+  ) => Promise<void>;
+  ftsDeleteDocument: (
+    id: string,
+    indexName: string,
+    docId: string
+  ) => Promise<void>;
+  ftsSearch: (
+    id: string,
+    indexName: string,
+    query: string
+  ) => Promise<{ id: string; score: number }[]>;
+  ftsGetDocument: (
+    id: string,
+    indexName: string,
+    docId: string
+  ) => Promise<string | null>;
+  ftsGetMatches: (
+    id: string,
+    indexName: string,
+    docId: string,
+    query: string
+  ) => Promise<{ start: number; end: number }[]>;
+  ftsFlushIndex: (id: string) => Promise<void>;
 }
 
 type NativeDBApisWrapper = NativeDBApis extends infer APIs

packages/common/nbstore/src/impls/sqlite/index.ts

@@ -3,16 +3,19 @@ import { SqliteBlobStorage } from './blob';
 import { SqliteBlobSyncStorage } from './blob-sync';
 import { SqliteDocStorage } from './doc';
 import { SqliteDocSyncStorage } from './doc-sync';
+import { SqliteIndexerStorage } from './indexer';
 
 export * from './blob';
 export * from './blob-sync';
 export { bindNativeDBApis, type NativeDBApis } from './db';
 export * from './doc';
 export * from './doc-sync';
+export * from './indexer';
 
 export const sqliteStorages = [
   SqliteDocStorage,
   SqliteBlobStorage,
   SqliteDocSyncStorage,
   SqliteBlobSyncStorage,
+  SqliteIndexerStorage,
 ] satisfies StorageConstructor[];

packages/common/nbstore/src/impls/sqlite/indexer/index.ts

@@ -0,0 +1,244 @@
+import { merge, Observable, of, Subject } from 'rxjs';
+import { filter, throttleTime } from 'rxjs/operators';
+
+import { share } from '../../../connection';
+import type {
+  AggregateOptions,
+  AggregateResult,
+  IndexerDocument,
+  Query,
+  SearchOptions,
+  SearchResult,
+} from '../../../storage';
+import { IndexerStorageBase } from '../../../storage';
+import { IndexerSchema } from '../../../storage/indexer/schema';
+import { fromPromise } from '../../../utils/from-promise';
+import { backoffRetry, exhaustMapWithTrailing } from '../../idb/indexer/utils';
+import { NativeDBConnection, type SqliteNativeDBOptions } from '../db';
+import { createNode } from './node-builder';
+import { queryRaw } from './query';
+import { getText, tryParseArrayField } from './utils';
+
+export class SqliteIndexerStorage extends IndexerStorageBase {
+  static readonly identifier = 'SqliteIndexerStorage';
+  override readonly recommendRefreshInterval = 30 * 1000; // 5 seconds
+  readonly connection: NativeDBConnection;
+  readonly isReadonly = false;
+  private readonly tableUpdate$ = new Subject<string>();
+
+  constructor(options: SqliteNativeDBOptions) {
+    super();
+    this.connection = share(new NativeDBConnection(options));
+  }
+
+  private watchTableUpdated(table: string) {
+    return this.tableUpdate$.asObservable().pipe(filter(t => t === table));
+  }
+
+  async search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Promise<SearchResult<T, O>> {
+    const match = await queryRaw(this.connection, table, query);
+
+    // Pagination
+    const limit = options?.pagination?.limit ?? 10;
+    const skip = options?.pagination?.skip ?? 0;
+    const ids = match.toArray();
+    const pagedIds = ids.slice(skip, skip + limit);
+
+    const nodes = [];
+    for (const id of pagedIds) {
+      const node = await createNode(
+        this.connection,
+        table,
+        id,
+        match.getScore(id),
+        options ?? {},
+        query
+      );
+      nodes.push(node);
+    }
+
+    return {
+      pagination: {
+        count: ids.length,
+        limit,
+        skip,
+        hasMore: ids.length > skip + limit,
+      },
+      nodes,
+    };
+  }
+
+  async aggregate<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Promise<AggregateResult<T, O>> {
+    const match = await queryRaw(this.connection, table, query);
+    const ids = match.toArray();
+
+    const buckets: any[] = [];
+
+    for (const id of ids) {
+      const text = await this.connection.apis.ftsGetDocument(
+        `${table}:${field as string}`,
+        id
+      );
+      if (text) {
+        let values: string[] = [text];
+        const parsed = tryParseArrayField(text);
+        if (parsed) {
+          values = parsed;
+        }
+
+        for (const val of values) {
+          let bucket = buckets.find(b => b.key === val);
+          if (!bucket) {
+            bucket = { key: val, count: 0, score: 0 };
+            if (options?.hits) {
+              bucket.hits = {
+                pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
+                nodes: [],
+              };
+            }
+            buckets.push(bucket);
+          }
+          bucket.count++;
+
+          if (options?.hits) {
+            const hitLimit = options.hits.pagination?.limit ?? 3;
+            if (bucket.hits.nodes.length < hitLimit) {
+              const node = await createNode(
+                this.connection,
+                table,
+                id,
+                match.getScore(id),
+                options.hits,
+                query
+              );
+              bucket.hits.nodes.push(node);
+              bucket.hits.pagination.count++;
+            }
+          }
+        }
+      }
+    }
+
+    return {
+      pagination: {
+        count: buckets.length,
+        limit: 0,
+        skip: 0,
+        hasMore: false,
+      },
+      buckets,
+    };
+  }
+
+  search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
+    table: T,
+    query: Query<T>,
+    options?: O
+  ): Observable<SearchResult<T, O>> {
+    return merge(of(1), this.watchTableUpdated(table)).pipe(
+      throttleTime(3000, undefined, { leading: true, trailing: true }),
+      exhaustMapWithTrailing(() => {
+        return fromPromise(async () => {
+          return await this.search(table, query, options);
+        }).pipe(backoffRetry());
+      })
+    );
+  }
+
+  aggregate$<
+    T extends keyof IndexerSchema,
+    const O extends AggregateOptions<T>,
+  >(
+    table: T,
+    query: Query<T>,
+    field: keyof IndexerSchema[T],
+    options?: O
+  ): Observable<AggregateResult<T, O>> {
+    return merge(of(1), this.watchTableUpdated(table)).pipe(
+      throttleTime(3000, undefined, { leading: true, trailing: true }),
+      exhaustMapWithTrailing(() => {
+        return fromPromise(async () => {
+          return await this.aggregate(table, query, field, options);
+        }).pipe(backoffRetry());
+      })
+    );
+  }
+
+  async deleteByQuery<T extends keyof IndexerSchema>(
+    table: T,
+    query: Query<T>
+  ): Promise<void> {
+    const match = await queryRaw(this.connection, table, query);
+    const ids = match.toArray();
+    for (const id of ids) {
+      await this.delete(table, id);
+    }
+  }
+
+  async insert<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void> {
+    const schema = IndexerSchema[table];
+    for (const [field, values] of document.fields) {
+      const fieldSchema = schema[field];
+      // @ts-expect-error
+      const shouldIndex = fieldSchema.index !== false;
+      // @ts-expect-error
+      const shouldStore = fieldSchema.store !== false;
+
+      if (!shouldStore && !shouldIndex) continue;
+
+      const text = getText(values);
+
+      if (typeof text === 'string') {
+        await this.connection.apis.ftsAddDocument(
+          `${table}:${field as string}`,
+          document.id,
+          text,
+          shouldIndex
+        );
+      }
+    }
+    this.tableUpdate$.next(table);
+  }
+
+  async delete<T extends keyof IndexerSchema>(
+    table: T,
+    id: string
+  ): Promise<void> {
+    const schema = IndexerSchema[table];
+    for (const field of Object.keys(schema)) {
+      await this.connection.apis.ftsDeleteDocument(`${table}:${field}`, id);
+    }
+    this.tableUpdate$.next(table);
+  }
+
+  async update<T extends keyof IndexerSchema>(
+    table: T,
+    document: IndexerDocument<T>
+  ): Promise<void> {
+    // Update is essentially insert (overwrite)
+    return this.insert(table, document);
+  }
+
+  async refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
+    // No-op for memory index
+  }
+
+  async refreshIfNeed(): Promise<void> {
+    await this.connection.apis.ftsFlushIndex();
+  }
+}

packages/common/nbstore/src/impls/sqlite/indexer/match.ts

@@ -0,0 +1,105 @@
+export class Match {
+  scores = new Map<string, number>();
+  /**
+   * id -> field -> index(multi value field) -> [start, end][]
+   */
+  highlighters = new Map<
+    string,
+    Map<string, Map<number, [number, number][]>>
+  >();
+
+  constructor() {}
+
+  size() {
+    return this.scores.size;
+  }
+
+  getScore(id: string) {
+    return this.scores.get(id) ?? 0;
+  }
+
+  addScore(id: string, score: number) {
+    const currentScore = this.scores.get(id) || 0;
+    this.scores.set(id, currentScore + score);
+  }
+
+  getHighlighters(id: string, field: string) {
+    return this.highlighters.get(id)?.get(field);
+  }
+
+  addHighlighter(
+    id: string,
+    field: string,
+    index: number,
+    newRanges: [number, number][]
+  ) {
+    const fields =
+      this.highlighters.get(id) ||
+      new Map<string, Map<number, [number, number][]>>();
+    const values = fields.get(field) || new Map<number, [number, number][]>();
+    const ranges = values.get(index) || [];
+    ranges.push(...newRanges);
+    values.set(index, ranges);
+    fields.set(field, values);
+    this.highlighters.set(id, fields);
+  }
+
+  and(other: Match) {
+    const newMatch = new Match();
+    for (const [id, score] of this.scores) {
+      if (other.scores.has(id)) {
+        newMatch.addScore(id, score + (other.scores.get(id) ?? 0));
+        newMatch.copyExtData(this, id);
+        newMatch.copyExtData(other, id);
+      }
+    }
+    return newMatch;
+  }
+
+  or(other: Match) {
+    const newMatch = new Match();
+    for (const [id, score] of this.scores) {
+      newMatch.addScore(id, score);
+      newMatch.copyExtData(this, id);
+    }
+    for (const [id, score] of other.scores) {
+      newMatch.addScore(id, score);
+      newMatch.copyExtData(other, id);
+    }
+    return newMatch;
+  }
+
+  exclude(other: Match) {
+    const newMatch = new Match();
+    for (const [id, score] of this.scores) {
+      if (!other.scores.has(id)) {
+        newMatch.addScore(id, score);
+        newMatch.copyExtData(this, id);
+      }
+    }
+    return newMatch;
+  }
+
+  boost(boost: number) {
+    const newMatch = new Match();
+    for (const [id, score] of this.scores) {
+      newMatch.addScore(id, score * boost);
+      newMatch.copyExtData(this, id);
+    }
+    return newMatch;
+  }
+
+  toArray() {
+    return Array.from(this.scores.entries())
+      .sort((a, b) => b[1] - a[1])
+      .map(e => e[0]);
+  }
+
+  private copyExtData(from: Match, id: string) {
+    for (const [field, values] of from.highlighters.get(id) ?? []) {
+      for (const [index, ranges] of values) {
+        this.addHighlighter(id, field, index, ranges);
+      }
+    }
+  }
+}

packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts

@@ -0,0 +1,97 @@
+import { type Query, type SearchOptions } from '../../../storage';
+import { highlighter } from '../../idb/indexer/highlighter';
+import { type NativeDBConnection } from '../db';
+import { tryParseArrayField } from './utils';
+
+export async function createNode(
+  connection: NativeDBConnection,
+  table: string,
+  id: string,
+  score: number,
+  options: SearchOptions<any>,
+  query: Query<any>
+) {
+  const node: any = { id, score };
+
+  if (options.fields) {
+    const fields: Record<string, any> = {};
+    for (const field of options.fields) {
+      const text = await connection.apis.ftsGetDocument(
+        `${table}:${field as string}`,
+        id
+      );
+      if (text !== null) {
+        const parsed = tryParseArrayField(text);
+        if (parsed) {
+          fields[field as string] = parsed;
+        } else {
+          fields[field as string] = text;
+        }
+      } else {
+        fields[field as string] = '';
+      }
+    }
+    node.fields = fields;
+  }
+
+  if (options.highlights) {
+    const highlights: Record<string, string[]> = {};
+    const queryStrings = extractQueryStrings(query);
+
+    for (const h of options.highlights) {
+      const text = await connection.apis.ftsGetDocument(
+        `${table}:${h.field as string}`,
+        id
+      );
+      if (text) {
+        const queryString = Array.from(queryStrings).join(' ');
+        const matches = await connection.apis.ftsGetMatches(
+          `${table}:${h.field as string}`,
+          id,
+          queryString
+        );
+
+        if (matches.length > 0) {
+          const highlighted = highlighter(
+            text,
+            h.before,
+            h.end,
+            matches.map(m => [m.start, m.end]),
+            {
+              maxPrefix: 20,
+              maxLength: 50,
+            }
+          );
+          highlights[h.field as string] = highlighted ? [highlighted] : [];
+        } else {
+          highlights[h.field as string] = [];
+        }
+      } else {
+        highlights[h.field as string] = [];
+      }
+    }
+    node.highlights = highlights;
+  }
+
+  return node;
+}
+
+function extractQueryStrings(query: Query<any>): Set<string> {
+  const terms = new Set<string>();
+  if (query.type === 'match') {
+    terms.add(query.match);
+  } else if (query.type === 'boolean') {
+    for (const q of query.queries) {
+      const subTerms = extractQueryStrings(q);
+      for (const term of subTerms) {
+        terms.add(term);
+      }
+    }
+  } else if (query.type === 'boost') {
+    const subTerms = extractQueryStrings(query.query);
+    for (const term of subTerms) {
+      terms.add(term);
+    }
+  }
+  return terms;
+}

packages/common/nbstore/src/impls/sqlite/indexer/query.ts

@@ -0,0 +1,72 @@
+import { IndexerSchema, type Query } from '../../../storage';
+import { type NativeDBConnection } from '../db';
+import { Match } from './match';
+
+export async function queryRaw(
+  connection: NativeDBConnection,
+  table: string,
+  query: Query<any>
+): Promise<Match> {
+  if (query.type === 'match') {
+    const indexName = `${table}:${String(query.field)}`;
+    const hits = await connection.apis.ftsSearch(indexName, query.match);
+    const match = new Match();
+    for (const hit of hits ?? []) {
+      match.addScore(hit.id, hit.score);
+    }
+    return match;
+  } else if (query.type === 'boolean') {
+    const matches: Match[] = [];
+    for (const q of query.queries) {
+      matches.push(await queryRaw(connection, table, q));
+    }
+
+    if (query.occur === 'must') {
+      if (matches.length === 0) return new Match();
+      return matches.reduce((acc, m) => acc.and(m));
+    } else if (query.occur === 'should') {
+      if (matches.length === 0) return new Match();
+      return matches.reduce((acc, m) => acc.or(m));
+    } else if (query.occur === 'must_not') {
+      const union = matches.reduce((acc, m) => acc.or(m), new Match());
+      const all = await matchAll(connection, table);
+      return all.exclude(union);
+    }
+  } else if (query.type === 'all') {
+    return matchAll(connection, table);
+  } else if (query.type === 'boost') {
+    const match = await queryRaw(connection, table, query.query);
+    return match.boost(query.boost);
+  } else if (query.type === 'exists') {
+    const indexName = `${table}:${String(query.field)}`;
+    const hits = await connection.apis.ftsSearch(indexName, '*');
+    const match = new Match();
+    for (const hit of hits ?? []) {
+      match.addScore(hit.id, 1);
+    }
+    return match;
+  }
+
+  return new Match();
+}
+
+export async function matchAll(
+  connection: NativeDBConnection,
+  table: string
+): Promise<Match> {
+  const schema = IndexerSchema[table as keyof IndexerSchema];
+  if (!schema) return new Match();
+
+  const match = new Match();
+  for (const field of Object.keys(schema)) {
+    const indexName = `${table}:${field}`;
+    let hits = await connection.apis.ftsSearch(indexName, '');
+    if (!hits || hits.length === 0) {
+      hits = await connection.apis.ftsSearch(indexName, '*');
+    }
+    for (const hit of hits ?? []) {
+      match.addScore(hit.id, 1);
+    }
+  }
+  return match;
+}

packages/common/nbstore/src/impls/sqlite/indexer/utils.ts

@@ -0,0 +1,20 @@
+export function getText(
+  val: string | string[] | undefined
+): string | undefined {
+  if (Array.isArray(val)) {
+    return JSON.stringify(val);
+  }
+  return val;
+}
+
+export function tryParseArrayField(text: string): any[] | null {
+  if (text.startsWith('[') && text.endsWith(']')) {
+    try {
+      const parsed = JSON.parse(text);
+      if (Array.isArray(parsed)) {
+        return parsed;
+      }
+    } catch {}
+  }
+  return null;
+}

packages/common/nbstore/src/storage/indexer.ts

@@ -14,6 +14,7 @@ import type { Storage } from './storage';
 export interface IndexerStorage extends Storage {
   readonly storageType: 'indexer';
   readonly isReadonly: boolean;
+  readonly recommendRefreshInterval: number; // 100ms
 
   search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
     table: T,
@@ -120,6 +121,7 @@ export type AggregateResult<
 
 export abstract class IndexerStorageBase implements IndexerStorage {
   readonly storageType = 'indexer';
+  readonly recommendRefreshInterval: number = 100; // 100ms
   abstract readonly connection: Connection;
   abstract readonly isReadonly: boolean;
 

packages/common/nbstore/src/sync/indexer/index.ts

@@ -109,7 +109,7 @@ export class IndexerSyncImpl implements IndexerSync {
   /**
    * increase this number to re-index all docs
    */
-  readonly INDEXER_VERSION = 1;
+  readonly INDEXER_VERSION = 2;
   private abort: AbortController | null = null;
   private readonly rootDocId = this.doc.spaceId;
   private readonly status = new IndexerSyncStatus(this.rootDocId);
@@ -484,9 +484,16 @@ export class IndexerSyncImpl implements IndexerSync {
     }
   }
 
+  // ensure the indexer is refreshed according to recommendRefreshInterval
+  // recommendRefreshInterval <= 0 means force refresh on each operation
+  // recommendRefreshInterval > 0 means refresh if the last refresh is older than recommendRefreshInterval
   private async refreshIfNeed(): Promise<void> {
-    if (this.lastRefreshed + 100 < Date.now()) {
-      console.log('[indexer] refreshing indexer');
+    const recommendRefreshInterval = this.indexer.recommendRefreshInterval ?? 0;
+    const needRefresh =
+      recommendRefreshInterval > 0 &&
+      this.lastRefreshed + recommendRefreshInterval < Date.now();
+    const forceRefresh = recommendRefreshInterval <= 0;
+    if (needRefresh || forceRefresh) {
       await this.indexer.refreshIfNeed();
       this.lastRefreshed = Date.now();
     }