refactor(server): indexer & worker & sync perf (#15504)

packages/common/nbstore/src/__tests__/cloud-doc-updates.spec.ts

@@ -1,16 +1,51 @@
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 
+import { CloudAwarenessStorage } from '../impls/cloud/awareness';
 import { CloudDocStorage } from '../impls/cloud/doc';
 
 const base64UpdateA = 'AQID';
 const base64UpdateB = 'BAUG';
 
+class FakeSocket {
+  connected = true;
+  readonly emitted: Array<{ event: string; payload: unknown }> = [];
+  readonly handlers = new Map<string, (...args: unknown[]) => void>();
+
+  on(event: string, handler: (...args: unknown[]) => void) {
+    this.handlers.set(event, handler);
+    return this;
+  }
+
+  once(event: string, handler: (...args: unknown[]) => void) {
+    this.handlers.set(event, handler);
+    return this;
+  }
+
+  off(event: string, handler?: (...args: unknown[]) => void) {
+    if (!handler || this.handlers.get(event) === handler) {
+      this.handlers.delete(event);
+    }
+    return this;
+  }
+
+  emit(event: string, payload?: unknown) {
+    this.emitted.push({ event, payload });
+    return true;
+  }
+
+  async emitWithAck(event: string, payload: unknown) {
+    this.emitted.push({ event, payload });
+    return { data: { clientId: 'client-1', success: true } };
+  }
+}
+
 describe('CloudDocStorage broadcast updates', () => {
   test('emits updates from batch payload', () => {
     const storage = new CloudDocStorage({
       id: 'space-1',
       serverBaseUrl: 'http://localhost',
       isSelfHosted: true,
+      syncProtocol: 'legacy',
       type: 'workspace',
       readonlyMode: true,
     });
@@ -38,4 +73,164 @@ describe('CloudDocStorage broadcast updates', () => {
       new Uint8Array([4, 5, 6]),
     ]);
   });
+
+  test('repairs strict invalidation through readable timestamps', async () => {
+    const storage = new CloudDocStorage({
+      id: 'space-1',
+      serverBaseUrl: 'http://localhost',
+      isSelfHosted: true,
+      syncProtocol: 'batch',
+      type: 'workspace',
+      readonlyMode: true,
+    });
+
+    (storage as any).connection.idConverter = {
+      oldIdToNewId: (id: string) => id,
+      newIdToOldId: (id: string) => id,
+    };
+
+    const getDocTimestamps = vi
+      .spyOn(storage, 'getDocTimestamps')
+      .mockResolvedValue({ 'doc-a': new Date(1_000) });
+    const received: Array<{ docId: string; bin: Uint8Array }> = [];
+    storage.subscribeDocUpdate(update => {
+      received.push({ docId: update.docId, bin: update.bin });
+    });
+
+    storage.onServerInvalidation({
+      spaceType: 'workspace',
+      spaceId: 'space-1',
+      timestamp: 1_000,
+    });
+    storage.onServerInvalidation({
+      spaceType: 'workspace',
+      spaceId: 'space-1',
+      timestamp: 1_001,
+    });
+
+    await vi.waitFor(() => expect(received).toHaveLength(1));
+    expect(getDocTimestamps).toHaveBeenCalledOnce();
+    expect(received[0]).toMatchObject({ docId: 'doc-a' });
+    expect(received[0]?.bin).toEqual(new Uint8Array());
+  });
+
+  test.each([
+    ['legacy', 'space:join'],
+    ['batch', 'space:join-batch'],
+  ] as const)(
+    '%s route sends its own join event',
+    async (syncProtocol, event) => {
+      vi.stubGlobal('BUILD_CONFIG', { appVersion: '0.27.5' });
+      const fakeSocket = new FakeSocket();
+      const disconnect = vi.fn();
+      const storage = new CloudDocStorage({
+        id: 'space-1',
+        serverBaseUrl: 'http://localhost',
+        isSelfHosted: true,
+        syncProtocol,
+        type: 'workspace',
+        readonlyMode: true,
+      });
+      const connection = storage.connection as any;
+
+      Object.defineProperty(connection, 'manager', {
+        configurable: true,
+        value: {
+          connect: () => ({ socket: fakeSocket, disconnect }),
+        },
+      });
+      vi.spyOn(connection, 'getIdConverter').mockResolvedValue({
+        oldIdToNewId: (id: string) => id,
+        newIdToOldId: (id: string) => id,
+      });
+
+      const inner = await connection.doConnect();
+      expect(fakeSocket.emitted[0]?.event).toBe(event);
+
+      inner.disconnect();
+      vi.unstubAllGlobals();
+    }
+  );
+
+  test.each([
+    ['legacy', 'space:join-awareness'],
+    ['batch', 'space:join-batch'],
+  ] as const)(
+    '%s awareness joins for active documents',
+    async (syncProtocol, event) => {
+      vi.stubGlobal('BUILD_CONFIG', { appVersion: '0.27.5' });
+      const fakeSocket = new FakeSocket();
+      const storage = new CloudAwarenessStorage({
+        id: 'space-1',
+        serverBaseUrl: 'http://localhost',
+        isSelfHosted: true,
+        syncProtocol,
+        type: 'workspace',
+      });
+
+      Object.defineProperty(storage, 'connection', {
+        configurable: true,
+        value: {
+          status: 'connected',
+          inner: { socket: fakeSocket },
+          onStatusChanged: () => () => {},
+        },
+      });
+
+      const unsubscribeA = storage.subscribeUpdate(
+        'doc-a',
+        () => {},
+        async () => null
+      );
+      const unsubscribeB = storage.subscribeUpdate(
+        'doc-b',
+        () => {},
+        async () => null
+      );
+
+      await vi.waitFor(() => {
+        expect(
+          fakeSocket.emitted.filter(
+            ({ event: emittedEvent }) => emittedEvent === event
+          )
+        ).toHaveLength(syncProtocol === 'batch' ? 1 : 2);
+      });
+
+      if (syncProtocol === 'batch') {
+        expect(fakeSocket.emitted).toContainEqual({
+          event,
+          payload: {
+            spaces: [
+              { spaceType: 'workspace', spaceId: 'space-1', docId: 'doc-a' },
+              { spaceType: 'workspace', spaceId: 'space-1', docId: 'doc-b' },
+            ],
+            clientVersion: '0.27.5',
+          },
+        });
+      } else {
+        expect(fakeSocket.emitted).toContainEqual({
+          event,
+          payload: {
+            spaceType: 'workspace',
+            spaceId: 'space-1',
+            docId: 'doc-a',
+            clientVersion: '0.27.5',
+          },
+        });
+        expect(fakeSocket.emitted).toContainEqual({
+          event,
+          payload: {
+            spaceType: 'workspace',
+            spaceId: 'space-1',
+            docId: 'doc-b',
+            clientVersion: '0.27.5',
+          },
+        });
+      }
+
+      unsubscribeA();
+      unsubscribeB();
+      vi.unstubAllGlobals();
+    }
+  );
 });

packages/common/nbstore/src/__tests__/sync.spec.ts

@@ -694,6 +694,7 @@ test('indexer defers indexed clock persistence until a refresh happens on delaye
     })
   );
   const indexer = new TrackingIndexerStorage(calls, 30_000);
+  const update = vi.spyOn(indexer, 'update');
   const indexerSyncStorage = new TrackingIndexerSyncStorage(calls);
   const sync = new IndexerSyncImpl(
     docStorage,
@@ -712,6 +713,15 @@ test('indexer defers indexed clock persistence until a refresh happens on delaye
     sync.start();
     await sync.waitForCompleted();
 
+    const docUpdate = update.mock.calls.find(([table]) => table === 'doc');
+    expect(docUpdate).toBeDefined();
+    expect([...(docUpdate?.[1].fields ?? [])]).toEqual(
+      expect.arrayContaining([
+        ['docId', ['doc1']],
+        ['title', ['Doc 1']],
+        ['summary', ['summary']],
+      ])
+    );
     expect(calls).not.toContain('setClock:doc1');
 
     sync.stop();

packages/common/nbstore/src/impls/cloud/awareness.ts

@@ -6,12 +6,15 @@ import type { SpaceType } from '../../utils/universal-id';
 import {
   base64ToUint8Array,
   SocketConnection,
+  SPACE_JOIN_BATCH_LIMIT,
+  type SyncProtocol,
   uint8ArrayToBase64,
 } from './socket';
 
 interface CloudAwarenessStorageOptions {
   isSelfHosted: boolean;
   serverBaseUrl: string;
+  syncProtocol: SyncProtocol;
   type: SpaceType;
   id: string;
 }
@@ -32,6 +35,97 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
     return this.connection.inner.socket;
   }
 
+  private readonly activeAwarenessIds = new Set<string>();
+  private readonly joinedAwarenessIds = new Set<string>();
+  private joinPromise: Promise<void> | undefined;
+
+  private joinActiveAwareness(): Promise<void> {
+    if (
+      this.connection.status !== 'connected' ||
+      this.activeAwarenessIds.size === 0
+    ) {
+      return Promise.resolve();
+    }
+
+    if (this.joinPromise) {
+      return this.joinPromise;
+    }
+
+    const batchPromise = (async () => {
+      while (this.connection.status === 'connected') {
+        await Promise.resolve();
+        const pendingIds = [...this.activeAwarenessIds].filter(
+          docId => !this.joinedAwarenessIds.has(docId)
+        );
+        if (pendingIds.length === 0) {
+          return;
+        }
+
+        if (this.options.syncProtocol === 'batch') {
+          for (
+            let index = 0;
+            index < pendingIds.length;
+            index += SPACE_JOIN_BATCH_LIMIT
+          ) {
+            const spaces = pendingIds
+              .slice(index, index + SPACE_JOIN_BATCH_LIMIT)
+              .map(docId => ({
+                spaceType: this.options.type,
+                spaceId: this.options.id,
+                docId,
+              }));
+            const response = await this.socket.emitWithAck('space:join-batch', {
+              spaces,
+              clientVersion: BUILD_CONFIG.appVersion,
+            });
+
+            if ('error' in response) {
+              throw new Error(
+                `Awareness join failed: ${response.error.name}: ${response.error.message}`
+              );
+            }
+            if (!response.data.success) {
+              throw new Error('Awareness join was rejected');
+            }
+          }
+        } else {
+          for (const docId of pendingIds) {
+            const response = await this.socket.emitWithAck(
+              'space:join-awareness',
+              {
+                spaceType: this.options.type,
+                spaceId: this.options.id,
+                docId,
+                clientVersion: BUILD_CONFIG.appVersion,
+              }
+            );
+
+            if ('error' in response) {
+              throw new Error(
+                `Awareness join failed: ${response.error.name}: ${response.error.message}`
+              );
+            }
+            if (!response.data.success) {
+              throw new Error('Awareness join was rejected');
+            }
+          }
+        }
+
+        for (const docId of pendingIds) {
+          if (this.activeAwarenessIds.has(docId)) {
+            this.joinedAwarenessIds.add(docId);
+          }
+        }
+      }
+    })();
+
+    const sharedPromise = batchPromise.finally(() => {
+      this.joinPromise = undefined;
+    });
+    this.joinPromise = sharedPromise;
+    return sharedPromise;
+  }
+
   override async update(record: AwarenessRecord): Promise<void> {
     const encodedUpdate = await uint8ArrayToBase64(record.bin);
     this.socket.emit('space:update-awareness', {
@@ -47,19 +141,31 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
     onUpdate: (update: AwarenessRecord, origin?: string) => void,
     onCollect: () => Promise<AwarenessRecord | null>
   ): () => void {
+    this.activeAwarenessIds.add(id);
+
     // leave awareness
     const leave = () => {
-      if (this.connection.status !== 'connected') return;
+      this.activeAwarenessIds.delete(id);
+      this.joinedAwarenessIds.delete(id);
       this.socket.off('space:collect-awareness', handleCollectAwareness);
       this.socket.off(
         'space:broadcast-awareness-update',
         handleBroadcastAwarenessUpdate
       );
-      this.socket.emit('space:leave-awareness', {
-        spaceType: this.options.type,
-        spaceId: this.options.id,
-        docId: id,
-      });
+      if (this.connection.status !== 'connected') return;
+      if (this.options.syncProtocol === 'batch') {
+        this.socket.emit('space:leave-batch', {
+          spaceType: this.options.type,
+          spaceId: this.options.id,
+          docIds: [id],
+        });
+      } else {
+        this.socket.emit('space:leave-awareness', {
+          spaceType: this.options.type,
+          spaceId: this.options.id,
+          docId: id,
+        });
+      }
     };
 
     // join awareness, and collect awareness from others
@@ -69,12 +175,8 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
         'space:broadcast-awareness-update',
         handleBroadcastAwarenessUpdate
       );
-      await this.socket.emitWithAck('space:join-awareness', {
-        spaceType: this.options.type,
-        spaceId: this.options.id,
-        docId: id,
-        clientVersion: BUILD_CONFIG.appVersion,
-      });
+      await this.joinActiveAwareness();
+      if (this.connection.status !== 'connected') return;
       this.socket.emit('space:load-awarenesses', {
         spaceType: this.options.type,
         spaceId: this.options.id,
@@ -142,6 +244,9 @@ export class CloudAwarenessStorage extends AwarenessStorageBase {
 
     const unsubscribeConnectionStatusChanged = this.connection.onStatusChanged(
       status => {
+        if (status !== 'connected') {
+          this.joinedAwarenessIds.clear();
+        }
         if (status === 'connected') {
           joinAndCollect().catch(err =>
             console.error('awareness join failed', err)

packages/common/nbstore/src/impls/cloud/doc.ts

@@ -13,12 +13,14 @@ import {
   base64ToUint8Array,
   type ServerEventsMap,
   SocketConnection,
+  type SyncProtocol,
   uint8ArrayToBase64,
 } from './socket';
 
 interface CloudDocStorageOptions extends DocStorageOptions {
   serverBaseUrl: string;
   isSelfHosted: boolean;
+  syncProtocol: SyncProtocol;
   type: SpaceType;
 }
 
@@ -42,22 +44,6 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
   }
   readonly spaceType = this.options.type;
 
-  onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {
-    if (
-      this.spaceType !== message.spaceType ||
-      this.spaceId !== message.spaceId
-    ) {
-      return;
-    }
-
-    this.emit('update', {
-      docId: this.idConverter.oldIdToNewId(message.docId),
-      bin: base64ToUint8Array(message.update),
-      timestamp: new Date(message.timestamp),
-      editor: message.editor,
-    });
-  };
-
   onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'] = message => {
     if (
       this.spaceType !== message.spaceType ||
@@ -66,9 +52,11 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
       return;
     }
 
+    const docId = this.idConverter.oldIdToNewId(message.docId);
+    this.serverDocTimestamps.set(docId, message.timestamp);
     for (const update of message.updates) {
       this.emit('update', {
-        docId: this.idConverter.oldIdToNewId(message.docId),
+        docId,
         bin: base64ToUint8Array(update),
         timestamp: new Date(message.timestamp),
         editor: message.editor,
@@ -76,10 +64,70 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
     }
   };
 
+  onServerInvalidation: ServerEventsMap['space:broadcast-doc-invalidation'] =
+    message => {
+      if (
+        this.options.syncProtocol !== 'batch' ||
+        this.spaceType !== message.spaceType ||
+        this.spaceId !== message.spaceId
+      ) {
+        return;
+      }
+
+      this.invalidationPending = true;
+      this.scheduleInvalidationRepair();
+    };
+
+  private readonly serverDocTimestamps = new Map<string, number>();
+  private invalidationTimer?: ReturnType<typeof setTimeout>;
+  private invalidationRepair?: Promise<void>;
+  private invalidationPending = false;
+
+  private scheduleInvalidationRepair() {
+    if (this.invalidationTimer) {
+      clearTimeout(this.invalidationTimer);
+    }
+    this.invalidationTimer = setTimeout(() => {
+      this.invalidationTimer = undefined;
+      if (this.invalidationRepair) {
+        return;
+      }
+      this.invalidationPending = false;
+      this.invalidationRepair = this.repairInvalidatedDocs()
+        .catch(error => {
+          console.error('failed to repair invalidated docs', error);
+        })
+        .finally(() => {
+          this.invalidationRepair = undefined;
+          if (this.invalidationPending) {
+            this.scheduleInvalidationRepair();
+          }
+        });
+    }, 50);
+  }
+
+  private async repairInvalidatedDocs() {
+    const timestamps = await this.getDocTimestamps();
+    for (const [docId, timestamp] of Object.entries(timestamps)) {
+      const newDocId = this.idConverter.oldIdToNewId(docId);
+      const timestampValue = timestamp.getTime();
+      const previous = this.serverDocTimestamps.get(newDocId);
+      if (previous !== undefined && previous >= timestampValue) {
+        continue;
+      }
+      this.serverDocTimestamps.set(newDocId, timestampValue);
+      this.emit('update', {
+        docId: newDocId,
+        bin: new Uint8Array(),
+        timestamp,
+      });
+    }
+  }
+
   readonly connection = new CloudDocStorageConnection(
     this.options,
-    this.onServerUpdate,
-    this.onServerUpdates
+    this.onServerUpdates,
+    this.onServerInvalidation
   );
 
   override async getDocSnapshot(docId: string) {
@@ -216,8 +264,8 @@ export class CloudDocStorage extends DocStorageBase<CloudDocStorageOptions> {
 class CloudDocStorageConnection extends SocketConnection {
   constructor(
     private readonly options: CloudDocStorageOptions,
-    private readonly onServerUpdate: ServerEventsMap['space:broadcast-doc-update'],
-    private readonly onServerUpdates: ServerEventsMap['space:broadcast-doc-updates']
+    private readonly onServerUpdates: ServerEventsMap['space:broadcast-doc-updates'],
+    private readonly onServerInvalidation: ServerEventsMap['space:broadcast-doc-invalidation']
   ) {
     super(options.serverBaseUrl, options.isSelfHosted);
   }
@@ -228,22 +276,36 @@ class CloudDocStorageConnection extends SocketConnection {
     const { socket, disconnect } = await super.doConnect(signal);
 
     try {
-      const res = await socket.emitWithAck('space:join', {
-        spaceType: this.options.type,
-        spaceId: this.options.id,
-        clientVersion: BUILD_CONFIG.appVersion,
-      });
+      const res =
+        this.options.syncProtocol === 'batch'
+          ? await socket.emitWithAck('space:join-batch', {
+              spaces: [
+                {
+                  spaceType: this.options.type,
+                  spaceId: this.options.id,
+                },
+              ],
+              clientVersion: BUILD_CONFIG.appVersion,
+            })
+          : await socket.emitWithAck('space:join', {
+              spaceType: this.options.type,
+              spaceId: this.options.id,
+              clientVersion: BUILD_CONFIG.appVersion,
+            });
 
       if ('error' in res) {
         throw createWebsocketError(res.error);
       }
+      if (!res.data.success) {
+        throw new Error('Space join was rejected');
+      }
 
       if (!this.idConverter) {
         this.idConverter = await this.getIdConverter(socket);
       }
 
-      socket.on('space:broadcast-doc-update', this.onServerUpdate);
       socket.on('space:broadcast-doc-updates', this.onServerUpdates);
+      socket.on('space:broadcast-doc-invalidation', this.onServerInvalidation);
 
       return { socket, disconnect };
     } catch (e) {
@@ -259,12 +321,20 @@ class CloudDocStorageConnection extends SocketConnection {
     socket: Socket;
     disconnect: () => void;
   }) {
-    socket.emit('space:leave', {
-      spaceType: this.options.type,
-      spaceId: this.options.id,
-    });
-    socket.off('space:broadcast-doc-update', this.onServerUpdate);
+    if (this.options.syncProtocol === 'batch') {
+      socket.emit('space:leave-batch', {
+        spaceType: this.options.type,
+        spaceId: this.options.id,
+        docIds: [],
+      });
+    } else {
+      socket.emit('space:leave', {
+        spaceType: this.options.type,
+        spaceId: this.options.id,
+      });
+    }
     socket.off('space:broadcast-doc-updates', this.onServerUpdates);
+    socket.off('space:broadcast-doc-invalidation', this.onServerInvalidation);
     super.doDisconnect({ socket, disconnect });
   }
 

packages/common/nbstore/src/impls/cloud/socket.ts

@@ -28,14 +28,6 @@ type WebsocketResponse<T> =
     };
 
 interface ServerEvents {
-  'space:broadcast-doc-update': {
-    spaceType: string;
-    spaceId: string;
-    docId: string;
-    update: string;
-    timestamp: number;
-    editor: string;
-  };
   'space:broadcast-doc-updates': {
     spaceType: string;
     spaceId: string;
@@ -45,6 +37,11 @@ interface ServerEvents {
     editor?: string;
     compressed?: boolean;
   };
+  'space:broadcast-doc-invalidation': {
+    spaceType: string;
+    spaceId: string;
+    timestamp: number;
+  };
 
   'space:collect-awareness': {
     spaceType: string;
@@ -62,11 +59,29 @@ interface ServerEvents {
   'realtime:event': RealtimeEvent;
 }
 
+export type SyncProtocol = 'legacy' | 'batch';
+
 interface ClientEvents {
   'space:join': [
     { spaceType: string; spaceId: string; clientVersion: string },
-    { clientId: string },
+    { clientId: string; success: boolean },
   ];
+  'space:join-batch': [
+    {
+      spaces: Array<{
+        spaceType: string;
+        spaceId: string;
+        docId?: string;
+      }>;
+      clientVersion: string;
+    },
+    { clientId: string; success: boolean },
+  ];
+  'space:leave-batch': {
+    spaceType: string;
+    spaceId: string;
+    docIds: string[];
+  };
   'space:leave': { spaceType: string; spaceId: string };
   'space:join-awareness': [
     {
@@ -75,7 +90,7 @@ interface ClientEvents {
       docId: string;
       clientVersion: string;
     },
-    { clientId: string },
+    { clientId: string; success: boolean },
   ];
   'space:leave-awareness': {
     spaceType: string;
@@ -133,6 +148,8 @@ interface ClientEvents {
   'realtime:unsubscribe': [RealtimeUnsubscribeEnvelope, { ok: true }];
 }
 
+export const SPACE_JOIN_BATCH_LIMIT = 100;
+
 export type ServerEventsMap = {
   [Key in keyof ServerEvents]: (data: ServerEvents[Key]) => void;
 };

packages/common/nbstore/src/impls/sqlite/db.ts

@@ -15,6 +15,42 @@ export interface SqliteNativeDBOptions {
   readonly id: string;
 }
 
+export interface NativeIndexField {
+  field: string;
+  values: string[];
+}
+
+export interface NativeIndexQuery {
+  kind: 'match' | 'exists' | 'all' | 'boolean' | 'boost';
+  field?: string;
+  value?: string;
+  occur?: 'must' | 'should' | 'must_not';
+  clauses?: NativeIndexQuery[];
+  boost?: number;
+}
+
+export interface NativeIndexSearchOptions {
+  limit: number;
+  offset: number;
+  fields: string[];
+  highlights: string[];
+}
+
+export interface NativeIndexHit {
+  id: string;
+  score: number;
+  fields: NativeIndexField[];
+  highlights: {
+    field: string;
+    values: { valueIndex: number; spans: { start: number; end: number }[] }[];
+  }[];
+}
+
+export interface NativeIndexSearchResult {
+  total: number;
+  hits: NativeIndexHit[];
+}
+
 export interface NativeDBApis {
   connect: (id: string) => Promise<void>;
   disconnect: (id: string) => Promise<void>;
@@ -40,6 +76,7 @@ export interface NativeDBApis {
     indexedClock: Date,
     indexerVersion: number
   ) => Promise<void>;
+  setDocIndexedClocks: (id: string, clocks: DocIndexedClock[]) => Promise<void>;
   clearDocIndexedClock: (id: string, docId: string) => Promise<void>;
   getBlob: (id: string, key: string) => Promise<BlobRecord | null>;
   setBlob: (id: string, blob: BlobRecord) => Promise<void>;
@@ -95,36 +132,42 @@ export interface NativeDBApis {
     blobId: string
   ) => Promise<Date | null>;
   crawlDocData: (id: string, docId: string) => Promise<CrawlResult>;
-  ftsAddDocument: (
+  indexUpsert: (
     id: string,
-    indexName: string,
-    docId: string,
-    text: string,
-    index: boolean
+    table: string,
+    document: { id: string; fields: NativeIndexField[] }
   ) => Promise<void>;
-  ftsDeleteDocument: (
+  indexDelete: (id: string, table: string, docId: string) => Promise<void>;
+  indexSearch: (
     id: string,
-    indexName: string,
-    docId: string
-  ) => Promise<void>;
-  ftsSearch: (
+    table: string,
+    query: NativeIndexQuery,
+    options: NativeIndexSearchOptions
+  ) => Promise<NativeIndexSearchResult>;
+  indexAggregate: (
     id: string,
-    indexName: string,
-    query: string
-  ) => Promise<{ id: string; score: number; terms: Array<string> }[]>;
-  ftsGetDocument: (
+    table: string,
+    query: NativeIndexQuery,
+    field: string,
+    limit: number,
+    offset: number,
+    hits?: NativeIndexSearchOptions
+  ) => Promise<{
+    total: number;
+    buckets: {
+      key: string;
+      count: number;
+      score: number;
+      hits: NativeIndexHit[];
+    }[];
+  }>;
+  indexDeleteByQuery: (
     id: string,
-    indexName: string,
-    docId: string
-  ) => Promise<string | null>;
-  ftsGetMatches: (
-    id: string,
-    indexName: string,
-    docId: string,
-    query: string
-  ) => Promise<{ start: number; end: number }[]>;
-  ftsFlushIndex: (id: string) => Promise<void>;
-  ftsIndexVersion: () => Promise<number>;
+    table: string,
+    query: NativeIndexQuery
+  ) => Promise<number>;
+  indexFlush: (id: string) => Promise<void>;
+  indexVersion: () => Promise<number>;
 }
 
 type NativeDBApisWrapper = NativeDBApis extends infer APIs

packages/common/nbstore/src/impls/sqlite/index.ts

@@ -8,7 +8,15 @@ import { SqliteIndexerSyncStorage } from './indexer-sync';
 
 export * from './blob';
 export * from './blob-sync';
-export { bindNativeDBApis, type NativeDBApis } from './db';
+export {
+  bindNativeDBApis,
+  type NativeDBApis,
+  type NativeIndexField,
+  type NativeIndexHit,
+  type NativeIndexQuery,
+  type NativeIndexSearchOptions,
+  type NativeIndexSearchResult,
+} from './db';
 export * from './doc';
 export * from './doc-sync';
 export * from './indexer';

packages/common/nbstore/src/impls/sqlite/indexer-sync.ts

@@ -7,6 +7,7 @@ import { NativeDBConnection, type SqliteNativeDBOptions } from './db';
 
 export class SqliteIndexerSyncStorage extends IndexerSyncStorageBase {
   static readonly identifier = 'SqliteIndexerSyncStorage';
+  override readonly commitsIndexAtomically = true;
 
   override connection = share(new NativeDBConnection(this.options));
 
@@ -32,6 +33,10 @@ export class SqliteIndexerSyncStorage extends IndexerSyncStorageBase {
     );
   }
 
+  override async setDocIndexedClocks(clocks: DocIndexedClock[]): Promise<void> {
+    await this.db.setDocIndexedClocks(clocks);
+  }
+
   override async clearDocIndexedClock(docId: string): Promise<void> {
     await this.db.clearDocIndexedClock(docId);
   }

packages/common/nbstore/src/impls/sqlite/indexer/index.ts

@@ -7,18 +7,21 @@ import type {
   AggregateOptions,
   AggregateResult,
   IndexerDocument,
+  IndexerSchema,
   Query,
   SearchOptions,
   SearchResult,
 } from '../../../storage';
 import { IndexerStorageBase } from '../../../storage';
-import { IndexerSchema } from '../../../storage/indexer/schema';
 import { fromPromise } from '../../../utils/from-promise';
 import { backoffRetry, exhaustMapWithTrailing } from '../../idb/indexer/utils';
-import { NativeDBConnection, type SqliteNativeDBOptions } from '../db';
+import {
+  NativeDBConnection,
+  type NativeIndexQuery,
+  type NativeIndexSearchOptions,
+  type SqliteNativeDBOptions,
+} from '../db';
 import { createNode } from './node-builder';
-import { queryRaw } from './query';
-import { getText, tryParseArrayField } from './utils';
 
 const SQLITE_INDEXER_VERSION_OFFSET = 1;
 
@@ -43,33 +46,21 @@ export class SqliteIndexerStorage extends IndexerStorageBase {
     query: Query<T>,
     options?: O
   ): Promise<SearchResult<T, O>> {
-    const match = await queryRaw(this.connection, table, query);
-
-    // Pagination
     const limit = options?.pagination?.limit ?? 10;
     const skip = options?.pagination?.skip ?? 0;
-    const ids = match.toArray();
-    const pagedIds = ids.slice(skip, skip + limit);
-
-    const nodes = [];
-    for (const id of pagedIds) {
-      const node = await createNode(
-        this.connection,
-        table,
-        id,
-        match.getScore(id),
-        options ?? {},
-        query
-      );
-      nodes.push(node);
-    }
+    const result = await this.connection.apis.indexSearch(
+      String(table),
+      toNativeQuery(query),
+      toNativeOptions(options, limit, skip)
+    );
+    const nodes = result.hits.map(hit => createNode(hit, options ?? {}));
 
     return {
       pagination: {
-        count: ids.length,
+        count: result.total,
         limit,
         skip,
-        hasMore: ids.length > skip + limit,
+        hasMore: result.total > skip + limit,
       },
       nodes,
     };
@@ -84,72 +75,49 @@ export class SqliteIndexerStorage extends IndexerStorageBase {
     field: keyof IndexerSchema[T],
     options?: O
   ): Promise<AggregateResult<T, O>> {
-    const match = await queryRaw(this.connection, table, query);
-    const ids = match.toArray();
-
-    const buckets: any[] = [];
-
-    for (const id of ids) {
-      const text = await this.connection.apis.ftsGetDocument(
-        `${table}:${field as string}`,
-        id
-      );
-      if (typeof text === 'string' && text.length > 0) {
-        let values: string[] = [text];
-        const parsed = tryParseArrayField(text);
-        if (parsed) {
-          values = parsed;
-        }
-
-        for (const val of values) {
-          let bucket = buckets.find(b => b.key === val);
-          if (!bucket) {
-            bucket = { key: val, count: 0, score: 0 };
-            if (options?.hits) {
-              bucket.hits = {
-                pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
-                nodes: [],
-              };
-            }
-            buckets.push(bucket);
+    const limit = options?.pagination?.limit ?? 10;
+    const skip = options?.pagination?.skip ?? 0;
+    const hitLimit = options?.hits?.pagination?.limit ?? 3;
+    const hitSkip = options?.hits?.pagination?.skip ?? 0;
+    const result = await this.connection.apis.indexAggregate(
+      String(table),
+      toNativeQuery(query),
+      String(field),
+      limit,
+      skip,
+      options?.hits
+        ? toNativeOptions(options.hits, hitLimit, hitSkip)
+        : undefined
+    );
+    const hitsOptions = options?.hits;
+    const buckets = result.buckets.map(bucket => ({
+      key: bucket.key,
+      count: bucket.count,
+      score: bucket.score,
+      ...(hitsOptions
+        ? {
+            hits: {
+              pagination: {
+                count: bucket.count,
+                limit: hitLimit,
+                skip: hitSkip,
+                hasMore: bucket.count > hitSkip + hitLimit,
+              },
+              nodes: bucket.hits.map(hit => createNode(hit, hitsOptions)),
+            },
           }
-          bucket.count++;
-
-          if (options?.hits) {
-            const hitLimit = options.hits.pagination?.limit ?? 3;
-            if (bucket.hits.nodes.length < hitLimit) {
-              const node = await createNode(
-                this.connection,
-                table,
-                id,
-                match.getScore(id),
-                options.hits,
-                query
-              );
-              bucket.hits.nodes.push(node);
-              bucket.hits.pagination.count++;
-            }
-          }
-        }
-      } else if (text != null && typeof text !== 'string') {
-        console.warn('[nbstore] invalid indexed aggregate type', {
-          table,
-          field: field as string,
-          id,
-          type: typeof text,
-        });
-      }
-    }
+        : {}),
+    }));
 
     return {
       pagination: {
-        count: buckets.length,
-        limit: 0,
-        skip: 0,
-        hasMore: false,
+        count: result.total,
+        limit,
+        skip,
+        hasMore: result.total > skip + limit,
       },
       buckets,
-    };
+    } as AggregateResult<T, O>;
   }
 
   search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
@@ -190,38 +158,23 @@ export class SqliteIndexerStorage extends IndexerStorageBase {
     table: T,
     query: Query<T>
   ): Promise<void> {
-    const match = await queryRaw(this.connection, table, query);
-    const ids = match.toArray();
-    for (const id of ids) {
-      await this.delete(table, id);
-    }
+    await this.connection.apis.indexDeleteByQuery(
+      String(table),
+      toNativeQuery(query)
+    );
   }
 
   async insert<T extends keyof IndexerSchema>(
     table: T,
     document: IndexerDocument<T>
   ): Promise<void> {
-    const schema = IndexerSchema[table];
-    for (const [field, values] of document.fields) {
-      const fieldSchema = schema[field];
-      // @ts-expect-error -- IndexerSchema uses runtime-keyed fields from each table schema.
-      const shouldIndex = fieldSchema.index !== false;
-      // @ts-expect-error -- IndexerSchema uses runtime-keyed fields from each table schema.
-      const shouldStore = fieldSchema.store !== false;
-
-      if (!shouldStore && !shouldIndex) continue;
-
-      const text = getText(values);
-
-      if (typeof text === 'string') {
-        await this.connection.apis.ftsAddDocument(
-          `${table}:${field as string}`,
-          document.id,
-          text,
-          shouldIndex
-        );
-      }
-    }
+    await this.connection.apis.indexUpsert(String(table), {
+      id: document.id,
+      fields: [...document.fields].map(([field, values]) => ({
+        field: String(field),
+        values,
+      })),
+    });
     this.tableUpdate$.next(table);
   }
 
@@ -229,10 +182,7 @@ export class SqliteIndexerStorage extends IndexerStorageBase {
     table: T,
     id: string
   ): Promise<void> {
-    const schema = IndexerSchema[table];
-    for (const field of Object.keys(schema)) {
-      await this.connection.apis.ftsDeleteDocument(`${table}:${field}`, id);
-    }
+    await this.connection.apis.indexDelete(String(table), id);
     this.tableUpdate$.next(table);
   }
 
@@ -249,13 +199,52 @@ export class SqliteIndexerStorage extends IndexerStorageBase {
   }
 
   async refreshIfNeed(): Promise<void> {
-    await this.connection.apis.ftsFlushIndex();
+    await this.connection.apis.indexFlush();
   }
 
   async indexVersion(): Promise<number> {
     return (
-      (await this.connection.apis.ftsIndexVersion()) +
+      (await this.connection.apis.indexVersion()) +
       SQLITE_INDEXER_VERSION_OFFSET
     );
   }
 }
+
+function toNativeQuery(query: Query<any>): NativeIndexQuery {
+  switch (query.type) {
+    case 'match':
+      return { kind: 'match', field: String(query.field), value: query.match };
+    case 'exists':
+      return { kind: 'exists', field: String(query.field) };
+    case 'all':
+      return { kind: 'all' };
+    case 'boolean':
+      return {
+        kind: 'boolean',
+        occur: query.occur,
+        clauses: query.queries.map(toNativeQuery),
+      };
+    case 'boost':
+      return {
+        kind: 'boost',
+        boost: query.boost,
+        clauses: [toNativeQuery(query.query)],
+      };
+  }
+}
+
+function toNativeOptions(
+  options: SearchOptions<any> | undefined,
+  limit: number,
+  offset: number
+): NativeIndexSearchOptions {
+  const highlights = options?.highlights?.map(item => String(item.field)) ?? [];
+  return {
+    limit,
+    offset,
+    fields: [
+      ...new Set([...(options?.fields?.map(String) ?? []), ...highlights]),
+    ],
+    highlights,
+  };
+}

packages/common/nbstore/src/impls/sqlite/indexer/match.ts

@@ -1,105 +0,0 @@
-export class Match {
-  scores = new Map<string, number>();
-  /**
-   * id -> field -> index(multi value field) -> [start, end][]
-   */
-  highlighters = new Map<
-    string,
-    Map<string, Map<number, [number, number][]>>
-  >();
-
-  constructor() {}
-
-  size() {
-    return this.scores.size;
-  }
-
-  getScore(id: string) {
-    return this.scores.get(id) ?? 0;
-  }
-
-  addScore(id: string, score: number) {
-    const currentScore = this.scores.get(id) || 0;
-    this.scores.set(id, currentScore + score);
-  }
-
-  getHighlighters(id: string, field: string) {
-    return this.highlighters.get(id)?.get(field);
-  }
-
-  addHighlighter(
-    id: string,
-    field: string,
-    index: number,
-    newRanges: [number, number][]
-  ) {
-    const fields =
-      this.highlighters.get(id) ||
-      new Map<string, Map<number, [number, number][]>>();
-    const values = fields.get(field) || new Map<number, [number, number][]>();
-    const ranges = values.get(index) || [];
-    ranges.push(...newRanges);
-    values.set(index, ranges);
-    fields.set(field, values);
-    this.highlighters.set(id, fields);
-  }
-
-  and(other: Match) {
-    const newMatch = new Match();
-    for (const [id, score] of this.scores) {
-      if (other.scores.has(id)) {
-        newMatch.addScore(id, score + (other.scores.get(id) ?? 0));
-        newMatch.copyExtData(this, id);
-        newMatch.copyExtData(other, id);
-      }
-    }
-    return newMatch;
-  }
-
-  or(other: Match) {
-    const newMatch = new Match();
-    for (const [id, score] of this.scores) {
-      newMatch.addScore(id, score);
-      newMatch.copyExtData(this, id);
-    }
-    for (const [id, score] of other.scores) {
-      newMatch.addScore(id, score);
-      newMatch.copyExtData(other, id);
-    }
-    return newMatch;
-  }
-
-  exclude(other: Match) {
-    const newMatch = new Match();
-    for (const [id, score] of this.scores) {
-      if (!other.scores.has(id)) {
-        newMatch.addScore(id, score);
-        newMatch.copyExtData(this, id);
-      }
-    }
-    return newMatch;
-  }
-
-  boost(boost: number) {
-    const newMatch = new Match();
-    for (const [id, score] of this.scores) {
-      newMatch.addScore(id, score * boost);
-      newMatch.copyExtData(this, id);
-    }
-    return newMatch;
-  }
-
-  toArray() {
-    return Array.from(this.scores.entries())
-      .sort((a, b) => b[1] - a[1])
-      .map(e => e[0]);
-  }
-
-  private copyExtData(from: Match, id: string) {
-    for (const [field, values] of from.highlighters.get(id) ?? []) {
-      for (const [index, ranges] of values) {
-        this.addHighlighter(id, field, index, ranges);
-      }
-    }
-  }
-}

packages/common/nbstore/src/impls/sqlite/indexer/node-builder.spec.ts

@@ -1,70 +1,35 @@
-import { describe, expect, it, vi } from 'vitest';
+import { describe, expect, it } from 'vitest';
 
-import type { NativeDBConnection } from '../db';
-import { SqliteIndexerStorage } from '.';
+import type { NativeIndexHit } from '../db';
 import { createNode } from './node-builder';
-import { getText } from './utils';
 
-const query = { type: 'match', field: 'title', match: 'query' } as const;
+function hit(fields: NativeIndexHit['fields']): NativeIndexHit {
+  return { id: 'doc-id', score: 1, fields, highlights: [] };
+}
 
-const connectionWith = (value: unknown) =>
-  ({
-    apis: {
-      ftsGetDocument: vi.fn().mockResolvedValue(value),
-      ftsGetMatches: vi.fn().mockResolvedValue([]),
-    },
-  }) as unknown as NativeDBConnection;
-
-describe('sqlite indexer node fields', () => {
+describe('sqlite indexer native result mapping', () => {
   it.each([
-    ['string', 'summary', 'summary'],
-    ['singleton array', ['one'], 'one'],
+    ['string', ['summary'], 'summary'],
     ['array', ['one', 'two'], ['one', 'two']],
-    ['serialized array', '["one","two"]', ['one', 'two']],
-    ['malformed array', '[not-json]', '[not-json]'],
-    ['null', null, ''],
-    ['missing', undefined, ''],
-    ['wrong type', 42, ''],
-  ])('isolates %s values', async (_, value, expected) => {
-    const node = await createNode(
-      connectionWith(Array.isArray(value) ? getText(value) : value),
-      'doc',
-      'doc-id',
-      1,
-      { fields: ['title'] },
-      query
-    );
-
+    ['missing', [], ''],
+  ])('maps %s stored values', (_, values, expected) => {
+    const node = createNode(hit([{ field: 'title', values }]), {
+      fields: ['title'],
+    });
     expect(node.fields.title).toEqual(expected);
   });
 
-  it('does not highlight non-string values', async () => {
-    const connection = connectionWith({ invalid: true });
-    const node = await createNode(
-      connection,
-      'doc',
-      'doc-id',
-      1,
-      { highlights: [{ field: 'title', before: '<b>', end: '</b>' }] },
-      query
-    );
-
-    expect(node.highlights.title).toEqual([]);
-    expect(connection.apis.ftsGetMatches).not.toHaveBeenCalled();
-  });
-
-  it('isolates wrong aggregate field types', async () => {
-    const connection = connectionWith(42);
-    connection.apis.ftsSearch = vi
-      .fn()
-      .mockResolvedValue([{ id: 'doc-id', score: 1, terms: [] }]);
-    const storage = Object.create(
-      SqliteIndexerStorage.prototype
-    ) as SqliteIndexerStorage;
-    Object.defineProperty(storage, 'connection', { value: connection });
-
-    await expect(
-      storage.aggregate('doc', query, 'title')
-    ).resolves.toMatchObject({ buckets: [] });
+  it('formats native highlight spans', () => {
+    const native = hit([{ field: 'title', values: ['hello search'] }]);
+    native.highlights = [
+      {
+        field: 'title',
+        values: [{ valueIndex: 0, spans: [{ start: 6, end: 12 }] }],
+      },
+    ];
+    const node = createNode(native, {
+      highlights: [{ field: 'title', before: '<b>', end: '</b>' }],
+    });
+    expect(node.highlights.title).toEqual(['hello <b>search</b>']);
   });
 });

packages/common/nbstore/src/impls/sqlite/indexer/node-builder.ts

@@ -1,113 +1,44 @@
-import { type Query, type SearchOptions } from '../../../storage';
+import { type SearchOptions } from '../../../storage';
 import { highlighter } from '../../idb/indexer/highlighter';
-import { type NativeDBConnection } from '../db';
-import { tryParseArrayField } from './utils';
+import type { NativeIndexHit } from '../db';
 
-export async function createNode(
-  connection: NativeDBConnection,
-  table: string,
-  id: string,
-  score: number,
-  options: SearchOptions<any>,
-  query: Query<any>
-) {
-  const node: any = { id, score };
+export function createNode(hit: NativeIndexHit, options: SearchOptions<any>) {
+  const node: any = { id: hit.id, score: hit.score };
+  const fields = new Map(hit.fields.map(field => [field.field, field.values]));
 
   if (options.fields) {
-    const fields: Record<string, any> = {};
-    for (const field of options.fields) {
-      const text = await connection.apis.ftsGetDocument(
-        `${table}:${field as string}`,
-        id
-      );
-      if (typeof text === 'string') {
-        const parsed = tryParseArrayField(text);
-        if (parsed) {
-          fields[field as string] = parsed;
-        } else {
-          fields[field as string] = text;
-        }
-      } else if (text == null) {
-        fields[field as string] = '';
-      } else {
-        console.warn('[nbstore] invalid indexed field type', {
-          table,
-          field: field as string,
-          id,
-          type: typeof text,
-        });
-        fields[field as string] = '';
-      }
-    }
-    node.fields = fields;
+    node.fields = Object.fromEntries(
+      options.fields.map(field => {
+        const values = fields.get(String(field)) ?? [];
+        return [String(field), values.length > 1 ? values : (values[0] ?? '')];
+      })
+    );
   }
 
   if (options.highlights) {
-    const highlights: Record<string, string[]> = {};
-    const queryStrings = extractQueryStrings(query);
-
-    for (const h of options.highlights) {
-      const text = await connection.apis.ftsGetDocument(
-        `${table}:${h.field as string}`,
-        id
-      );
-      if (typeof text === 'string' && text.length > 0) {
-        const queryString = Array.from(queryStrings).join(' ');
-        const matches = await connection.apis.ftsGetMatches(
-          `${table}:${h.field as string}`,
-          id,
-          queryString
-        );
-
-        if (matches.length > 0) {
-          const highlighted = highlighter(
+    const highlights = new Map(
+      hit.highlights.map(item => [item.field, item.values])
+    );
+    node.highlights = Object.fromEntries(
+      options.highlights.map(option => {
+        const field = String(option.field);
+        const source = fields.get(field) ?? [];
+        const fragments = (highlights.get(field) ?? []).flatMap(value => {
+          const text = source[value.valueIndex];
+          if (!text) return [];
+          const fragment = highlighter(
             text,
-            h.before,
-            h.end,
-            matches.map(m => [m.start, m.end]),
-            {
-              maxPrefix: 20,
-              maxLength: 50,
-            }
+            option.before,
+            option.end,
+            value.spans.map(span => [span.start, span.end]),
+            { maxPrefix: 20, maxLength: 50 }
           );
-          highlights[h.field as string] = highlighted ? [highlighted] : [];
-        } else {
-          highlights[h.field as string] = [];
-        }
-      } else {
-        if (text != null && typeof text !== 'string') {
-          console.warn('[nbstore] invalid indexed highlight type', {
-            table,
-            field: h.field as string,
-            id,
-            type: typeof text,
-          });
-        }
-        highlights[h.field as string] = [];
-      }
-    }
-    node.highlights = highlights;
+          return fragment ? [fragment] : [];
+        });
+        return [field, fragments];
+      })
+    );
   }
 
   return node;
 }
-
-function extractQueryStrings(query: Query<any>): Set<string> {
-  const terms = new Set<string>();
-  if (query.type === 'match') {
-    terms.add(query.match);
-  } else if (query.type === 'boolean') {
-    for (const q of query.queries) {
-      const subTerms = extractQueryStrings(q);
-      for (const term of subTerms) {
-        terms.add(term);
-      }
-    }
-  } else if (query.type === 'boost') {
-    const subTerms = extractQueryStrings(query.query);
-    for (const term of subTerms) {
-      terms.add(term);
-    }
-  }
-  return terms;
-}

packages/common/nbstore/src/impls/sqlite/indexer/query.ts

@@ -1,72 +0,0 @@
-import { IndexerSchema, type Query } from '../../../storage';
-import { type NativeDBConnection } from '../db';
-import { Match } from './match';
-
-export async function queryRaw(
-  connection: NativeDBConnection,
-  table: string,
-  query: Query<any>
-): Promise<Match> {
-  if (query.type === 'match') {
-    const indexName = `${table}:${String(query.field)}`;
-    const hits = await connection.apis.ftsSearch(indexName, query.match);
-    const match = new Match();
-    for (const hit of hits ?? []) {
-      match.addScore(hit.id, hit.score);
-    }
-    return match;
-  } else if (query.type === 'boolean') {
-    const matches: Match[] = [];
-    for (const q of query.queries) {
-      matches.push(await queryRaw(connection, table, q));
-    }
-
-    if (query.occur === 'must') {
-      if (matches.length === 0) return new Match();
-      return matches.reduce((acc, m) => acc.and(m));
-    } else if (query.occur === 'should') {
-      if (matches.length === 0) return new Match();
-      return matches.reduce((acc, m) => acc.or(m));
-    } else if (query.occur === 'must_not') {
-      const union = matches.reduce((acc, m) => acc.or(m), new Match());
-      const all = await matchAll(connection, table);
-      return all.exclude(union);
-    }
-  } else if (query.type === 'all') {
-    return matchAll(connection, table);
-  } else if (query.type === 'boost') {
-    const match = await queryRaw(connection, table, query.query);
-    return match.boost(query.boost);
-  } else if (query.type === 'exists') {
-    const indexName = `${table}:${String(query.field)}`;
-    const hits = await connection.apis.ftsSearch(indexName, '*');
-    const match = new Match();
-    for (const hit of hits ?? []) {
-      match.addScore(hit.id, 1);
-    }
-    return match;
-  }
-
-  return new Match();
-}
-
-export async function matchAll(
-  connection: NativeDBConnection,
-  table: string
-): Promise<Match> {
-  const schema = IndexerSchema[table as keyof IndexerSchema];
-  if (!schema) return new Match();
-
-  const match = new Match();
-  for (const field of Object.keys(schema)) {
-    const indexName = `${table}:${field}`;
-    let hits = await connection.apis.ftsSearch(indexName, '');
-    if (!hits || hits.length === 0) {
-      hits = await connection.apis.ftsSearch(indexName, '*');
-    }
-    for (const hit of hits ?? []) {
-      match.addScore(hit.id, 1);
-    }
-  }
-  return match;
-}

packages/common/nbstore/src/impls/sqlite/indexer/utils.ts

@@ -1,23 +0,0 @@
-export function getText(
-  val: string | string[] | undefined
-): string | undefined {
-  if (Array.isArray(val)) {
-    if (val.length === 1) {
-      return val[0];
-    }
-    return JSON.stringify(val);
-  }
-  return val;
-}
-
-export function tryParseArrayField(text: string): any[] | null {
-  if (text.startsWith('[') && text.endsWith(']')) {
-    try {
-      const parsed = JSON.parse(text);
-      if (Array.isArray(parsed)) {
-        return parsed;
-      }
-    } catch {}
-  }
-  return null;
-}

packages/common/nbstore/src/storage/indexer-sync.ts

@@ -8,18 +8,24 @@ export interface DocIndexedClock extends DocClock {
 
 export interface IndexerSyncStorage extends Storage {
   readonly storageType: 'indexerSync';
+  readonly commitsIndexAtomically: boolean;
 
   getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
 
   setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
+  setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise<void>;
 
   clearDocIndexedClock(docId: string): Promise<void>;
 }
 
 export abstract class IndexerSyncStorageBase implements IndexerSyncStorage {
   readonly storageType = 'indexerSync';
+  readonly commitsIndexAtomically: boolean = false;
   abstract connection: Connection<any>;
   abstract getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
   abstract setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
+  async setDocIndexedClocks(docClocks: DocIndexedClock[]): Promise<void> {
+    for (const clock of docClocks) await this.setDocIndexedClock(clock);
+  }
   abstract clearDocIndexedClock(docId: string): Promise<void>;
 }

packages/common/nbstore/src/sync/doc/peer.ts

@@ -553,13 +553,16 @@ export class DocSyncPeer {
       this.actions.addDoc(docId);
       this.actions.updateRemoteClock(docId, remoteClock);
 
-      // schedule push job
-      this.schedule({
-        type: 'save',
-        docId,
-        remoteClock: remoteClock,
-        update,
-      });
+      if (isEmptyUpdate(update)) {
+        this.schedule({ type: 'pull', docId });
+      } else {
+        this.schedule({
+          type: 'save',
+          docId,
+          remoteClock: remoteClock,
+          update,
+        });
+      }
     },
   };
 

packages/common/nbstore/src/sync/indexer/index.ts

@@ -349,9 +349,13 @@ export class IndexerSyncImpl implements IndexerSync {
                   IndexerDocument.from(docId, {
                     docId,
                     title,
+                    summary: existingDoc.summary,
                   })
                 );
-                this.status.docsInIndexer.set(docId, { title });
+                this.status.docsInIndexer.set(docId, {
+                  title,
+                  summary: existingDoc.summary,
+                });
                 this.status.statusUpdatedSubject$.next(docId);
               }
             } else {
@@ -461,9 +465,15 @@ export class IndexerSyncImpl implements IndexerSync {
             await this.indexer.update(
               'doc',
               IndexerDocument.from(docId, {
+                docId,
+                title: existingDoc.title,
                 summary: preview,
               })
             );
+            this.status.docsInIndexer.set(docId, {
+              title: existingDoc.title,
+              summary: preview,
+            });
           }
 
           this.pendingIndexedClocks.set(docId, {
@@ -496,18 +506,21 @@ export class IndexerSyncImpl implements IndexerSync {
       this.lastRefreshed + recommendRefreshInterval < Date.now();
     const forceRefresh = recommendRefreshInterval <= 0;
     if (force || needRefresh || forceRefresh) {
-      await this.indexer.refreshIfNeed();
-      await this.flushPendingIndexedClocks();
+      if (this.indexerSync.commitsIndexAtomically) {
+        await this.flushPendingIndexedClocks();
+      } else {
+        await this.indexer.refreshIfNeed();
+        await this.flushPendingIndexedClocks();
+      }
       this.lastRefreshed = Date.now();
     }
   }
 
   private async flushPendingIndexedClocks() {
     if (this.pendingIndexedClocks.size === 0) return;
-    for (const [docId, clock] of this.pendingIndexedClocks) {
-      await this.indexerSync.setDocIndexedClock(clock);
-      this.pendingIndexedClocks.delete(docId);
-    }
+    const clocks = [...this.pendingIndexedClocks.values()];
+    await this.indexerSync.setDocIndexedClocks(clocks);
+    for (const clock of clocks) this.pendingIndexedClocks.delete(clock.docId);
   }
 
   /**
@@ -559,16 +572,20 @@ export class IndexerSyncImpl implements IndexerSync {
         pagination: {
           limit: Infinity,
         },
-        fields: ['docId', 'title'],
+        fields: ['docId', 'title', 'summary'],
       }
     );
 
     return new Map(
       docs.nodes.map(node => {
         const title = node.fields.title;
+        const summary = node.fields.summary;
         return [
           node.id,
-          { title: typeof title === 'string' ? title : undefined },
+          {
+            title: typeof title === 'string' ? title : undefined,
+            summary: typeof summary === 'string' ? summary : undefined,
+          },
         ];
       })
     );
@@ -691,7 +708,10 @@ class IndexerSyncStatus {
   jobs = new AsyncPriorityQueue();
   rootDoc = new YDoc({ guid: this.rootDocId });
   rootDocReady = false;
-  docsInIndexer = new Map<string, { title: string | undefined }>();
+  docsInIndexer = new Map<
+    string,
+    { title: string | undefined; summary?: string }
+  >();
   docsInRootDoc = new Map<string, { title: string | undefined }>();
   currentJob: string | null = null;
   errorMessage: string | null = null;