feat(native): native reader for indexer (#14055)

packages/common/nbstore/src/impls/cloud/indexer.ts

@@ -139,4 +139,8 @@ export class CloudIndexerStorage extends IndexerStorageBase {
   override refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
     return Promise.resolve();
   }
+
+  override async refreshIfNeed(): Promise<void> {
+    return Promise.resolve();
+  }
 }

packages/common/nbstore/src/impls/idb/indexer/index.ts

@@ -176,6 +176,21 @@ export class IndexedDBIndexerStorage extends IndexerStorageBase {
     this.emitTableUpdated(table);
   }
 
+  override async refreshIfNeed(): Promise<void> {
+    const needRefreshTable = Object.entries(this.pendingUpdates)
+      .filter(
+        ([, updates]) =>
+          updates.deleteByQueries.length > 0 ||
+          updates.deletes.length > 0 ||
+          updates.inserts.length > 0 ||
+          updates.updates.length > 0
+      )
+      .map(([table]) => table as keyof IndexerSchema);
+    for (const table of needRefreshTable) {
+      await this.refresh(table);
+    }
+  }
+
   private watchTableUpdated(table: keyof IndexerSchema) {
     return new Observable(subscriber => {
       const listener = (ev: MessageEvent) => {

packages/common/nbstore/src/impls/sqlite/db.ts

@@ -1,6 +1,7 @@
 import { AutoReconnectConnection } from '../../connection';
 import type {
   BlobRecord,
+  CrawlResult,
   DocClock,
   DocRecord,
   ListedBlobRecord,
@@ -81,6 +82,7 @@ export interface NativeDBApis {
     peer: string,
     blobId: string
   ) => Promise<Date | null>;
+  crawlDocData: (id: string, docId: string) => Promise<CrawlResult>;
 }
 
 type NativeDBApisWrapper = NativeDBApis extends infer APIs

packages/common/nbstore/src/impls/sqlite/doc.ts

@@ -1,5 +1,7 @@
 import { share } from '../../connection';
 import {
+  type BlockInfo,
+  type CrawlResult,
   type DocClocks,
   type DocRecord,
   DocStorageBase,
@@ -79,4 +81,127 @@ export class SqliteDocStorage extends DocStorageBase<SqliteNativeDBOptions> {
       updates.map(update => update.timestamp)
     );
   }
+
+  override async crawlDocData(docId: string): Promise<CrawlResult | null> {
+    const result = await this.db.crawlDocData(docId);
+    return normalizeNativeCrawlResult(result);
+  }
+}
+
+function normalizeNativeCrawlResult(result: unknown): CrawlResult | null {
+  if (!isRecord(result)) {
+    console.warn('[nbstore] crawlDocData returned non-object result');
+    return null;
+  }
+
+  if (
+    typeof result.title !== 'string' ||
+    typeof result.summary !== 'string' ||
+    !Array.isArray(result.blocks)
+  ) {
+    console.warn('[nbstore] crawlDocData result missing basic fields');
+    return null;
+  }
+
+  const { title, summary } = result as { title: string; summary: string };
+  const rawBlocks = result.blocks as unknown[];
+
+  const blocks: BlockInfo[] = [];
+  for (const block of rawBlocks) {
+    const normalized = normalizeBlock(block);
+    if (normalized) {
+      blocks.push(normalized);
+    }
+  }
+
+  if (blocks.length === 0) {
+    console.warn('[nbstore] crawlDocData has no valid blocks');
+    return null;
+  }
+
+  return {
+    blocks,
+    title,
+    summary,
+  };
+}
+
+function normalizeBlock(block: unknown): BlockInfo | null {
+  if (!isRecord(block)) {
+    return null;
+  }
+
+  const blockId = readStringField(block, 'blockId');
+  const flavour = readStringField(block, 'flavour');
+
+  if (!blockId || !flavour) {
+    return null;
+  }
+
+  return {
+    blockId,
+    flavour,
+    content: readStringArrayField(block, 'content'),
+    blob: readStringArrayField(block, 'blob'),
+    refDocId: readStringArrayField(block, 'refDocId'),
+    refInfo: readStringArrayField(block, 'refInfo'),
+    parentFlavour: readStringField(block, 'parentFlavour'),
+    parentBlockId: readStringField(block, 'parentBlockId'),
+    additional: safeAdditionalField(block),
+  };
+}
+
+function readStringField(
+  target: Record<string, unknown>,
+  key: string
+): string | undefined {
+  const value = readField(target, key);
+  return typeof value === 'string' && value ? value : undefined;
+}
+
+function readStringArrayField(
+  target: Record<string, unknown>,
+  key: string
+): string[] | undefined {
+  const value = readField(target, key);
+  if (Array.isArray(value)) {
+    const filtered = value.filter(
+      (item): item is string => typeof item === 'string' && item.length > 0
+    );
+    return filtered.length ? filtered : undefined;
+  }
+  if (typeof value === 'string' && value.length > 0) {
+    return [value];
+  }
+  return undefined;
+}
+
+function safeAdditionalField(
+  target: Record<string, unknown>
+): string | undefined {
+  const value = readField(target, 'additional');
+  if (typeof value !== 'string' || value.length === 0) {
+    return undefined;
+  }
+  try {
+    const parsed = JSON.parse(value);
+    return JSON.stringify(parsed);
+  } catch {
+    console.warn(
+      '[nbstore] ignore invalid additional payload in crawlDocData block'
+    );
+    return undefined;
+  }
+}
+
+function readField(target: Record<string, unknown>, key: string) {
+  return target[key] ?? target[toSnakeCase(key)];
+}
+
+function toSnakeCase(key: string) {
+  return key.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
 }