refactor(server): indexer & worker & sync perf (#15504)

packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/plugin/NbStorePlugin.kt

@@ -7,9 +7,10 @@ import com.getcapacitor.PluginCall
 import com.getcapacitor.PluginMethod
 import com.getcapacitor.annotation.CapacitorPlugin
 import kotlinx.coroutines.Dispatchers
-import org.json.JSONObject
 import timber.log.Timber
 import uniffi.affine_mobile_native.DocRecord
+import uniffi.affine_mobile_native.DocIndexedClock
+import uniffi.affine_mobile_native.IndexHit
 import uniffi.affine_mobile_native.SetBlob
 import uniffi.affine_mobile_native.newDocStoragePool
 
@@ -623,119 +624,209 @@ class NbStorePlugin : Plugin() {
   }
 
   @PluginMethod
-  fun ftsAddDocument(call: PluginCall) {
+  fun getDocIndexedClock(call: PluginCall) {
     launch(Dispatchers.IO) {
       try {
-        val id = call.getStringEnsure("id")
-        val indexName = call.getStringEnsure("indexName")
-        val docId = call.getStringEnsure("docId")
-        val text = call.getStringEnsure("text")
-        val index = call.getBoolean("index")
-          ?: throw IllegalArgumentException("index is required")
-        docStoragePool.ftsAddDocument(id, indexName, docId, text, index)
-        call.resolve()
+        val clock = docStoragePool.getDocIndexedClock(
+          call.getStringEnsure("id"),
+          call.getStringEnsure("docId")
+        )
+        call.resolve(clock?.let(::indexedClockJson))
       } catch (e: Exception) {
-        call.reject("Failed to add document to fts: ${e.message}", null, e)
+        call.reject("Failed to get indexed clock: ${e.message}", null, e)
       }
     }
   }
 
   @PluginMethod
-  fun ftsDeleteDocument(call: PluginCall) {
+  fun setDocIndexedClock(call: PluginCall) {
     launch(Dispatchers.IO) {
       try {
-        val id = call.getStringEnsure("id")
-        val indexName = call.getStringEnsure("indexName")
-        val docId = call.getStringEnsure("docId")
-        docStoragePool.ftsDeleteDocument(id, indexName, docId)
+        val clock = DocIndexedClock(
+          call.getStringEnsure("docId"),
+          call.getLong("indexedClock") ?: throw IllegalArgumentException("indexedClock is required"),
+          call.getLong("indexerVersion") ?: throw IllegalArgumentException("indexerVersion is required")
+        )
+        docStoragePool.setDocIndexedClock(call.getStringEnsure("id"), clock)
         call.resolve()
       } catch (e: Exception) {
-        call.reject("Failed to delete document from fts: ${e.message}", null, e)
+        call.reject("Failed to set indexed clock: ${e.message}", null, e)
       }
     }
   }
 
   @PluginMethod
-  fun ftsSearch(call: PluginCall) {
+  fun setDocIndexedClocks(call: PluginCall) {
     launch(Dispatchers.IO) {
       try {
-        val id = call.getStringEnsure("id")
-        val indexName = call.getStringEnsure("indexName")
-        val query = call.getStringEnsure("query")
-        val results = docStoragePool.ftsSearch(id, indexName, query)
-        val mapped = results.map {
-          JSObject()
-            .put("id", it.id)
-            .put("score", it.score)
-            .put("terms", JSArray(it.terms))
+        val values = call.getArray("clocks") ?: throw IllegalArgumentException("clocks is required")
+        val clocks = (0 until values.length()).map { index ->
+          val value = values.getJSONObject(index)
+          DocIndexedClock(
+            value.getString("docId"),
+            value.getLong("timestamp"),
+            value.getLong("indexerVersion")
+          )
         }
+        docStoragePool.setDocIndexedClocks(call.getStringEnsure("id"), clocks)
+        call.resolve()
+      } catch (e: Exception) {
+        call.reject("Failed to commit indexed clocks: ${e.message}", null, e)
+      }
+    }
+  }
+
+  @PluginMethod
+  fun clearDocIndexedClock(call: PluginCall) {
+    launch(Dispatchers.IO) {
+      try {
+        docStoragePool.clearDocIndexedClock(call.getStringEnsure("id"), call.getStringEnsure("docId"))
+        call.resolve()
+      } catch (e: Exception) {
+        call.reject("Failed to clear indexed clock: ${e.message}", null, e)
+      }
+    }
+  }
+
+  @PluginMethod
+  fun indexUpsert(call: PluginCall) {
+    launch(Dispatchers.IO) {
+      try {
+        val id = call.getStringEnsure("id")
+        val table = call.getStringEnsure("table")
+        val document = call.getObject("document")
+          ?: throw IllegalArgumentException("document is required")
+        docStoragePool.indexUpsert(id, table, document.toString())
+        call.resolve()
+      } catch (e: Exception) {
+        call.reject("Failed to upsert index document: ${e.message}", null, e)
+      }
+    }
+  }
+
+  @PluginMethod
+  fun indexDelete(call: PluginCall) {
+    launch(Dispatchers.IO) {
+      try {
+        val id = call.getStringEnsure("id")
+        val table = call.getStringEnsure("table")
+        val docId = call.getStringEnsure("docId")
+        docStoragePool.indexDelete(id, table, docId)
+        call.resolve()
+      } catch (e: Exception) {
+        call.reject("Failed to delete index document: ${e.message}", null, e)
+      }
+    }
+  }
+
+  @PluginMethod
+  fun indexSearch(call: PluginCall) {
+    launch(Dispatchers.IO) {
+      try {
+        val id = call.getStringEnsure("id")
+        val table = call.getStringEnsure("table")
+        val query = call.getObject("query") ?: throw IllegalArgumentException("query is required")
+        val options = call.getObject("options") ?: throw IllegalArgumentException("options is required")
+        val result = docStoragePool.indexSearch(id, table, query.toString(), options.toString())
         call.resolve(
-          JSObject().put("results", JSArray(mapped))
+          JSObject()
+            .put("total", result.total.toInt())
+            .put("hits", JSArray(result.hits.map(::indexHitJson)))
         )
       } catch (e: Exception) {
-        call.reject("Failed to search fts: ${e.message}", null, e)
+        call.reject("Failed to search index: ${e.message}", null, e)
       }
     }
   }
 
   @PluginMethod
-  fun ftsGetDocument(call: PluginCall) {
+  fun indexAggregate(call: PluginCall) {
     launch(Dispatchers.IO) {
       try {
         val id = call.getStringEnsure("id")
-        val indexName = call.getStringEnsure("indexName")
-        val docId = call.getStringEnsure("docId")
-        val text = docStoragePool.ftsGetDocument(id, indexName, docId)
-        call.resolve(JSObject().put("text", text ?: JSONObject.NULL))
-      } catch (e: Exception) {
-        call.reject("Failed to get fts document: ${e.message}", null, e)
-      }
-    }
-  }
-
-  @PluginMethod
-  fun ftsGetMatches(call: PluginCall) {
-    launch(Dispatchers.IO) {
-      try {
-        val id = call.getStringEnsure("id")
-        val indexName = call.getStringEnsure("indexName")
-        val docId = call.getStringEnsure("docId")
-        val query = call.getStringEnsure("query")
-        val matches = docStoragePool.ftsGetMatches(id, indexName, docId, query)
-        val mapped = matches.map {
+        val table = call.getStringEnsure("table")
+        val query = call.getObject("query") ?: throw IllegalArgumentException("query is required")
+        val hits = call.getObject("hits")?.toString()
+        val result = docStoragePool.indexAggregate(
+          id,
+          table,
+          query.toString(),
+          call.getStringEnsure("field"),
+          call.getIntEnsure("limit").toUInt(),
+          call.getIntEnsure("offset").toUInt(),
+          hits
+        )
+        val buckets = result.buckets.map {
           JSObject()
-            .put("start", it.start.toInt())
-            .put("end", it.end.toInt())
+            .put("key", it.key)
+            .put("count", it.count.toInt())
+            .put("score", it.score)
+            .put("hits", JSArray(it.hits.map(::indexHitJson)))
         }
-        call.resolve(JSObject().put("matches", JSArray(mapped)))
+        call.resolve(JSObject().put("total", result.total.toInt()).put("buckets", JSArray(buckets)))
       } catch (e: Exception) {
-        call.reject("Failed to get fts matches: ${e.message}", null, e)
+        call.reject("Failed to aggregate index: ${e.message}", null, e)
       }
     }
   }
 
   @PluginMethod
-  fun ftsFlushIndex(call: PluginCall) {
+  fun indexDeleteByQuery(call: PluginCall) {
     launch(Dispatchers.IO) {
       try {
         val id = call.getStringEnsure("id")
-        docStoragePool.ftsFlushIndex(id)
+        val table = call.getStringEnsure("table")
+        val query = call.getObject("query") ?: throw IllegalArgumentException("query is required")
+        val deleted = docStoragePool.indexDeleteByQuery(id, table, query.toString())
+        call.resolve(JSObject().put("deleted", deleted.toInt()))
+      } catch (e: Exception) {
+        call.reject("Failed to delete index documents: ${e.message}", null, e)
+      }
+    }
+  }
+
+  @PluginMethod
+  fun indexFlush(call: PluginCall) {
+    launch(Dispatchers.IO) {
+      try {
+        docStoragePool.indexFlush(call.getStringEnsure("id"))
         call.resolve()
       } catch (e: Exception) {
-        call.reject("Failed to flush fts index: ${e.message}", null, e)
+        call.reject("Failed to flush index: ${e.message}", null, e)
       }
     }
   }
 
   @PluginMethod
-  fun ftsIndexVersion(call: PluginCall) {
+  fun indexVersion(call: PluginCall) {
     launch(Dispatchers.IO) {
       try {
-        val version = docStoragePool.ftsIndexVersion() + ANDROID_INDEXER_VERSION_OFFSET
+        val version = docStoragePool.indexVersion() + ANDROID_INDEXER_VERSION_OFFSET
         call.resolve(JSObject().put("indexVersion", version))
       } catch (e: Exception) {
-        call.reject("Failed to get fts index version: ${e.message}", null, e)
+        call.reject("Failed to get index version: ${e.message}", null, e)
       }
     }
   }
 }
+
+private fun indexHitJson(hit: IndexHit): JSObject {
+  val fields = hit.fields.map { JSObject().put("field", it.field).put("values", JSArray(it.values)) }
+  val highlights = hit.highlights.map { highlight ->
+    val values = highlight.values.map { value ->
+      val spans = value.spans.map { JSObject().put("start", it.start.toInt()).put("end", it.end.toInt()) }
+      JSObject().put("valueIndex", value.valueIndex.toInt()).put("spans", JSArray(spans))
+    }
+    JSObject().put("field", highlight.field).put("values", JSArray(values))
+  }
+  return JSObject()
+    .put("id", hit.id)
+    .put("score", hit.score)
+    .put("fields", JSArray(fields))
+    .put("highlights", JSArray(highlights))
+}
+
+private fun indexedClockJson(clock: DocIndexedClock): JSObject = JSObject()
+  .put("docId", clock.docId)
+  .put("timestamp", clock.timestamp)
+  .put("indexerVersion", clock.indexerVersion)

packages/frontend/apps/android/App/app/src/main/java/app/affine/pro/plugin/PluginExt.kt

@@ -25,4 +25,8 @@ inline fun <reified T> PluginCall.getListEnsure(key: String): List<T> {
 
 fun PluginCall.getLongEnsure(key: String): Long {
     return getLong(key) ?: throw IllegalArgumentException("Missing $key parameter")
-}
+}
+
+fun PluginCall.getIntEnsure(key: String): Int {
+    return getInt(key) ?: throw IllegalArgumentException("Missing $key parameter")
+}

packages/frontend/apps/android/src/plugins/nbstore/definitions.ts

@@ -1,4 +1,22 @@
 import type { CrawlResult, DocIndexedClock } from '@affine/nbstore';
+import type {
+  NativeIndexField,
+  NativeIndexHit,
+  NativeIndexQuery,
+  NativeIndexSearchOptions,
+  NativeIndexSearchResult,
+} from '@affine/nbstore/sqlite';
+
+type NativeIndexDocument = { id: string; fields: NativeIndexField[] };
+type NativeIndexAggregateResult = {
+  total: number;
+  buckets: {
+    key: string;
+    count: number;
+    score: number;
+    hits: NativeIndexHit[];
+  }[];
+};
 
 export interface Blob {
   key: string;
@@ -157,38 +175,38 @@ export interface NbStorePlugin {
     id: string;
     docId: string;
   }) => Promise<CrawlResult>;
-  ftsAddDocument: (options: {
+  indexUpsert: (options: {
     id: string;
-    indexName: string;
-    docId: string;
-    text: string;
-    index: boolean;
+    table: string;
+    document: NativeIndexDocument;
   }) => Promise<void>;
-  ftsDeleteDocument: (options: {
+  indexDelete: (options: {
     id: string;
-    indexName: string;
+    table: string;
     docId: string;
   }) => Promise<void>;
-  ftsSearch: (options: {
+  indexSearch: (options: {
     id: string;
-    indexName: string;
-    query: string;
-  }) => Promise<{
-    results: { id: string; score: number; terms: Array<string> }[];
-  }>;
-  ftsGetDocument: (options: {
+    table: string;
+    query: NativeIndexQuery;
+    options: NativeIndexSearchOptions;
+  }) => Promise<NativeIndexSearchResult>;
+  indexAggregate: (options: {
     id: string;
-    indexName: string;
-    docId: string;
-  }) => Promise<{ text?: string | null }>;
-  ftsGetMatches: (options: {
+    table: string;
+    query: NativeIndexQuery;
+    field: string;
+    limit: number;
+    offset: number;
+    hits?: NativeIndexSearchOptions;
+  }) => Promise<NativeIndexAggregateResult>;
+  indexDeleteByQuery: (options: {
     id: string;
-    indexName: string;
-    docId: string;
-    query: string;
-  }) => Promise<{ matches: { start: number; end: number }[] }>;
-  ftsFlushIndex: (options: { id: string }) => Promise<void>;
-  ftsIndexVersion: () => Promise<{ indexVersion: number }>;
+    table: string;
+    query: NativeIndexQuery;
+  }) => Promise<{ deleted: number }>;
+  indexFlush: (options: { id: string }) => Promise<void>;
+  indexVersion: () => Promise<{ indexVersion: number }>;
   getDocIndexedClock: (options: {
     id: string;
     docId: string;
@@ -199,6 +217,10 @@ export interface NbStorePlugin {
     indexedClock: number;
     indexerVersion: number;
   }) => Promise<void>;
+  setDocIndexedClocks: (options: {
+    id: string;
+    clocks: Array<{ docId: string; timestamp: number; indexerVersion: number }>;
+  }) => Promise<void>;
   clearDocIndexedClock: (options: {
     id: string;
     docId: string;

packages/frontend/apps/android/src/plugins/nbstore/index.ts

@@ -2,7 +2,6 @@ import {
   base64ToUint8Array,
   uint8ArrayToBase64,
 } from '@affine/core/modules/workspace-engine';
-import { normalizeNativeOptional } from '@affine/mobile-shared/nbstore/optional';
 import {
   decodePayload,
   MOBILE_BLOB_FILE_PREFIX,
@@ -358,77 +357,74 @@ export const NbStoreNativeDBApis: NativeDBApis = {
   ): Promise<CrawlResult> {
     return await NbStore.crawlDocData({ id, docId });
   },
-  ftsAddDocument: async function (
+  indexUpsert: async function (
     id: string,
-    indexName: string,
-    docId: string,
-    text: string,
-    index: boolean
+    table: string,
+    document
   ): Promise<void> {
-    await NbStore.ftsAddDocument({
+    await NbStore.indexUpsert({
       id,
-      indexName,
-      docId,
-      text,
-      index,
+      table,
+      document,
     });
   },
-  ftsDeleteDocument: async function (
+  indexDelete: async function (
     id: string,
-    indexName: string,
+    table: string,
     docId: string
   ): Promise<void> {
-    await NbStore.ftsDeleteDocument({
+    await NbStore.indexDelete({
       id,
-      indexName,
+      table,
       docId,
     });
   },
-  ftsSearch: async function (
-    id: string,
-    indexName: string,
-    query: string
-  ): Promise<{ id: string; score: number; terms: Array<string> }[]> {
-    const { results } = await NbStore.ftsSearch({
+  indexSearch: async function (id: string, table: string, query, options) {
+    return await NbStore.indexSearch({
       id,
-      indexName,
+      table,
+      query,
+      options,
+    });
+  },
+  indexAggregate: async function (
+    id: string,
+    table: string,
+    query,
+    field: string,
+    limit: number,
+    offset: number,
+    hits
+  ) {
+    return await NbStore.indexAggregate({
+      id,
+      table,
+      query,
+      field,
+      limit,
+      offset,
+      hits,
+    });
+  },
+  indexDeleteByQuery: async function (
+    id: string,
+    table: string,
+    query
+  ): Promise<number> {
+    const { deleted } = await NbStore.indexDeleteByQuery({
+      id,
+      table,
       query,
     });
-    return results ?? [];
+    return deleted;
   },
-  ftsGetDocument: async function (
-    id: string,
-    indexName: string,
-    docId: string
-  ): Promise<string | null> {
-    const result = await NbStore.ftsGetDocument({
-      id,
-      indexName,
-      docId,
-    });
-    return normalizeNativeOptional(result.text);
-  },
-  ftsGetMatches: async function (
-    id: string,
-    indexName: string,
-    docId: string,
-    query: string
-  ): Promise<{ start: number; end: number }[]> {
-    const { matches } = await NbStore.ftsGetMatches({
-      id,
-      indexName,
-      docId,
-      query,
-    });
-    return matches ?? [];
-  },
-  ftsFlushIndex: async function (id: string): Promise<void> {
-    await NbStore.ftsFlushIndex({
+  indexFlush: async function (id: string): Promise<void> {
+    await NbStore.indexFlush({
       id,
     });
   },
-  ftsIndexVersion: function (): Promise<number> {
-    return NbStore.ftsIndexVersion().then(res => res.indexVersion);
+  indexVersion: function (): Promise<number> {
+    return NbStore.indexVersion().then(res => res.indexVersion);
   },
   getDocIndexedClock: function (
     id: string,
@@ -451,6 +447,15 @@ export const NbStoreNativeDBApis: NativeDBApis = {
       indexerVersion,
     });
   },
+  setDocIndexedClocks: function (id, clocks): Promise<void> {
+    return NbStore.setDocIndexedClocks({
+      id,
+      clocks: clocks.map(clock => ({
+        ...clock,
+        timestamp: clock.timestamp.getTime(),
+      })),
+    });
+  },
   clearDocIndexedClock: function (id: string, docId: string): Promise<void> {
     return NbStore.clearDocIndexedClock({
       id,