refactor(server): indexer & worker & sync perf (#15504)

packages/frontend/mobile-native/Cargo.toml

@@ -22,6 +22,7 @@ affine_nbstore = { workspace = true }
 anyhow         = { workspace = true }
 base64-simd    = { workspace = true }
 chrono         = { workspace = true }
+serde_json     = { workspace = true }
 thiserror      = { workspace = true }
 tokio          = { workspace = true, features = ["rt-multi-thread"] }
 uniffi         = { workspace = true, features = ["cli", "tokio"] }

packages/frontend/mobile-native/src/error.rs

@@ -16,4 +16,10 @@ impl From<NbStoreError> for UniffiError {
   }
 }
 
+impl From<serde_json::Error> for UniffiError {
+  fn from(err: serde_json::Error) -> Self {
+    Self::Err(err.to_string())
+  }
+}
+
 pub(crate) type Result<T> = std::result::Result<T, UniffiError>;

packages/frontend/mobile-native/src/ffi_types.rs

@@ -1,7 +1,10 @@
 use affine_nbstore::{
-  Blob as NbBlob, Data, DocClock as NbDocClock, DocRecord as NbDocRecord, DocUpdate as NbDocUpdate,
-  ListedBlob as NbListedBlob, SetBlob as NbSetBlob,
-  indexer::{NativeBlockInfo, NativeCrawlResult, NativeMatch, NativeSearchHit},
+  Blob as NbBlob, Data, DocClock as NbDocClock, DocIndexedClock as NbDocIndexedClock, DocRecord as NbDocRecord,
+  DocUpdate as NbDocUpdate, ListedBlob as NbListedBlob, SetBlob as NbSetBlob,
+  indexer::{
+    NativeBlockInfo, NativeCrawlResult, NativeIndexAggregateResult, NativeIndexBucket, NativeIndexField,
+    NativeIndexHighlight, NativeIndexHighlightValue, NativeIndexHit, NativeIndexSearchResult, NativeIndexSpan,
+  },
 };
 use chrono::{DateTime, Utc};
 
@@ -80,6 +83,37 @@ pub struct DocClock {
   pub timestamp: i64,
 }
 
+#[derive(uniffi::Record)]
+pub struct DocIndexedClock {
+  pub doc_id: String,
+  pub timestamp: i64,
+  pub indexer_version: i64,
+}
+
+impl From<NbDocIndexedClock> for DocIndexedClock {
+  fn from(clock: NbDocIndexedClock) -> Self {
+    Self {
+      doc_id: clock.doc_id,
+      timestamp: clock.timestamp.and_utc().timestamp_millis(),
+      indexer_version: clock.indexer_version,
+    }
+  }
+}
+
+impl TryFrom<DocIndexedClock> for NbDocIndexedClock {
+  type Error = UniffiError;
+
+  fn try_from(clock: DocIndexedClock) -> Result<Self> {
+    Ok(Self {
+      doc_id: clock.doc_id,
+      timestamp: DateTime::<Utc>::from_timestamp_millis(clock.timestamp)
+        .ok_or(UniffiError::TimestampDecodingError)?
+        .naive_utc(),
+      indexer_version: clock.indexer_version,
+    })
+  }
+}
+
 impl From<NbDocClock> for DocClock {
   fn from(clock: NbDocClock) -> Self {
     Self {
@@ -211,33 +245,129 @@ impl From<NativeCrawlResult> for CrawlResult {
 }
 
 #[derive(uniffi::Record)]
-pub struct SearchHit {
-  pub id: String,
-  pub score: f64,
-  pub terms: Vec<String>,
+pub struct IndexField {
+  pub field: String,
+  pub values: Vec<String>,
 }
 
-impl From<NativeSearchHit> for SearchHit {
-  fn from(value: NativeSearchHit) -> Self {
+impl From<NativeIndexField> for IndexField {
+  fn from(value: NativeIndexField) -> Self {
     Self {
-      id: value.id,
-      score: value.score,
-      terms: value.terms,
+      field: value.field,
+      values: value.values,
     }
   }
 }
 
 #[derive(uniffi::Record)]
-pub struct MatchRange {
+pub struct IndexSpan {
   pub start: u32,
   pub end: u32,
 }
 
-impl From<NativeMatch> for MatchRange {
-  fn from(value: NativeMatch) -> Self {
+impl From<NativeIndexSpan> for IndexSpan {
+  fn from(value: NativeIndexSpan) -> Self {
     Self {
       start: value.start,
       end: value.end,
     }
   }
 }
+
+#[derive(uniffi::Record)]
+pub struct IndexHighlightValue {
+  pub value_index: u32,
+  pub spans: Vec<IndexSpan>,
+}
+
+impl From<NativeIndexHighlightValue> for IndexHighlightValue {
+  fn from(value: NativeIndexHighlightValue) -> Self {
+    Self {
+      value_index: value.value_index,
+      spans: value.spans.into_iter().map(Into::into).collect(),
+    }
+  }
+}
+
+#[derive(uniffi::Record)]
+pub struct IndexHighlight {
+  pub field: String,
+  pub values: Vec<IndexHighlightValue>,
+}
+
+impl From<NativeIndexHighlight> for IndexHighlight {
+  fn from(value: NativeIndexHighlight) -> Self {
+    Self {
+      field: value.field,
+      values: value.values.into_iter().map(Into::into).collect(),
+    }
+  }
+}
+
+#[derive(uniffi::Record)]
+pub struct IndexHit {
+  pub id: String,
+  pub score: f64,
+  pub fields: Vec<IndexField>,
+  pub highlights: Vec<IndexHighlight>,
+}
+
+impl From<NativeIndexHit> for IndexHit {
+  fn from(value: NativeIndexHit) -> Self {
+    Self {
+      id: value.id,
+      score: value.score,
+      fields: value.fields.into_iter().map(Into::into).collect(),
+      highlights: value.highlights.into_iter().map(Into::into).collect(),
+    }
+  }
+}
+
+#[derive(uniffi::Record)]
+pub struct IndexSearchResult {
+  pub total: u32,
+  pub hits: Vec<IndexHit>,
+}
+
+impl From<NativeIndexSearchResult> for IndexSearchResult {
+  fn from(value: NativeIndexSearchResult) -> Self {
+    Self {
+      total: value.total,
+      hits: value.hits.into_iter().map(Into::into).collect(),
+    }
+  }
+}
+
+#[derive(uniffi::Record)]
+pub struct IndexBucket {
+  pub key: String,
+  pub count: u32,
+  pub score: f64,
+  pub hits: Vec<IndexHit>,
+}
+
+impl From<NativeIndexBucket> for IndexBucket {
+  fn from(value: NativeIndexBucket) -> Self {
+    Self {
+      key: value.key,
+      count: value.count,
+      score: value.score,
+      hits: value.hits.into_iter().map(Into::into).collect(),
+    }
+  }
+}
+
+#[derive(uniffi::Record)]
+pub struct IndexAggregateResult {
+  pub total: u32,
+  pub buckets: Vec<IndexBucket>,
+}
+
+impl From<NativeIndexAggregateResult> for IndexAggregateResult {
+  fn from(value: NativeIndexAggregateResult) -> Self {
+    Self {
+      total: value.total,
+      buckets: value.buckets.into_iter().map(Into::into).collect(),
+    }
+  }
+}

packages/frontend/mobile-native/src/lib.rs

@@ -14,7 +14,8 @@ use affine_common::hashcash::Stamp;
 pub(crate) use error::Result;
 pub use error::UniffiError;
 pub use ffi_types::{
-  Blob, BlockInfo, CrawlResult, DocClock, DocRecord, DocUpdate, ListedBlob, MatchRange, SearchHit, SetBlob,
+  Blob, BlockInfo, CrawlResult, DocClock, DocIndexedClock, DocRecord, DocUpdate, IndexAggregateResult, IndexBucket,
+  IndexField, IndexHighlight, IndexHighlightValue, IndexHit, IndexSearchResult, IndexSpan, ListedBlob, SetBlob,
 };
 #[cfg(any(target_os = "android", target_os = "ios"))]
 pub use preview::{render_mermaid_preview_svg, render_typst_preview_svg};

packages/frontend/mobile-native/src/storage/indexer.rs

@@ -1,3 +1,5 @@
+use affine_nbstore::indexer::{NativeIndexDocument, NativeIndexQuery, NativeIndexSearchOptions};
+
 use super::*;
 
 #[uniffi::export(async_runtime = "tokio")]
@@ -12,89 +14,135 @@ impl DocStoragePool {
     Ok(result.into())
   }
 
-  pub async fn fts_add_document(
-    &self,
-    universal_id: String,
-    index_name: String,
-    doc_id: String,
-    text: String,
-    index: bool,
-  ) -> Result<()> {
+  pub async fn index_upsert(&self, universal_id: String, table: String, document: String) -> Result<()> {
+    let document: NativeIndexDocument = serde_json::from_str(&document)?;
     self
       .inner
       .get(universal_id)
       .await?
-      .fts_add(&index_name, &doc_id, &text, index)
+      .index_upsert(&table, document)
       .await?;
     Ok(())
   }
 
-  pub async fn fts_delete_document(&self, universal_id: String, index_name: String, doc_id: String) -> Result<()> {
+  pub async fn index_delete(&self, universal_id: String, table: String, doc_id: String) -> Result<()> {
     self
       .inner
       .get(universal_id)
       .await?
-      .fts_delete(&index_name, &doc_id)
+      .index_delete(&table, &doc_id)
       .await?;
     Ok(())
   }
 
-  pub async fn fts_get_document(
+  pub async fn index_search(
     &self,
     universal_id: String,
-    index_name: String,
-    doc_id: String,
-  ) -> Result<Option<String>> {
+    table: String,
+    query: String,
+    options: String,
+  ) -> Result<IndexSearchResult> {
+    let query: NativeIndexQuery = serde_json::from_str(&query)?;
+    let options: NativeIndexSearchOptions = serde_json::from_str(&options)?;
     Ok(
       self
         .inner
         .get(universal_id)
         .await?
-        .fts_get(&index_name, &doc_id)
+        .index_search(&table, query, options)
+        .await?
+        .into(),
+    )
+  }
+
+  #[allow(clippy::too_many_arguments)]
+  pub async fn index_aggregate(
+    &self,
+    universal_id: String,
+    table: String,
+    query: String,
+    field: String,
+    limit: u32,
+    offset: u32,
+    hits: Option<String>,
+  ) -> Result<IndexAggregateResult> {
+    let query: NativeIndexQuery = serde_json::from_str(&query)?;
+    let hits = hits
+      .map(|value| serde_json::from_str::<NativeIndexSearchOptions>(&value))
+      .transpose()?;
+    Ok(
+      self
+        .inner
+        .get(universal_id)
+        .await?
+        .index_aggregate(&table, query, &field, limit, offset, hits)
+        .await?
+        .into(),
+    )
+  }
+
+  pub async fn index_delete_by_query(&self, universal_id: String, table: String, query: String) -> Result<u32> {
+    let query: NativeIndexQuery = serde_json::from_str(&query)?;
+    Ok(
+      self
+        .inner
+        .get(universal_id)
+        .await?
+        .index_delete_by_query(&table, query)
         .await?,
     )
   }
 
-  pub async fn fts_search(&self, universal_id: String, index_name: String, query: String) -> Result<Vec<SearchHit>> {
-    Ok(
-      self
-        .inner
-        .get(universal_id)
-        .await?
-        .fts_search(&index_name, &query)
-        .await?
-        .into_iter()
-        .map(Into::into)
-        .collect(),
-    )
-  }
-
-  pub async fn fts_get_matches(
-    &self,
-    universal_id: String,
-    index_name: String,
-    doc_id: String,
-    query: String,
-  ) -> Result<Vec<MatchRange>> {
-    Ok(
-      self
-        .inner
-        .get(universal_id)
-        .await?
-        .fts_get_matches(&index_name, &doc_id, &query)
-        .await?
-        .into_iter()
-        .map(Into::into)
-        .collect(),
-    )
-  }
-
-  pub async fn fts_flush_index(&self, universal_id: String) -> Result<()> {
+  pub async fn index_flush(&self, universal_id: String) -> Result<()> {
     self.inner.get(universal_id).await?.flush_index().await?;
     Ok(())
   }
 
-  pub async fn fts_index_version(&self) -> Result<u32> {
+  pub async fn index_version(&self) -> Result<u32> {
     Ok(SqliteDocStorage::index_version())
   }
+
+  pub async fn set_doc_indexed_clocks(&self, universal_id: String, clocks: Vec<DocIndexedClock>) -> Result<()> {
+    let clocks = clocks.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?;
+    self
+      .inner
+      .get(universal_id)
+      .await?
+      .commit_indexed_clocks(&clocks)
+      .await?;
+    Ok(())
+  }
+
+  pub async fn get_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<Option<DocIndexedClock>> {
+    Ok(
+      self
+        .inner
+        .get(universal_id)
+        .await?
+        .get_doc_indexed_clock(doc_id)
+        .await?
+        .map(Into::into),
+    )
+  }
+
+  pub async fn set_doc_indexed_clock(&self, universal_id: String, clock: DocIndexedClock) -> Result<()> {
+    let clock = clock.try_into()?;
+    self
+      .inner
+      .get(universal_id)
+      .await?
+      .commit_indexed_clocks(&[clock])
+      .await?;
+    Ok(())
+  }
+
+  pub async fn clear_doc_indexed_clock(&self, universal_id: String, doc_id: String) -> Result<()> {
+    self
+      .inner
+      .get(universal_id)
+      .await?
+      .clear_doc_indexed_clock(doc_id)
+      .await?;
+    Ok(())
+  }
 }

packages/frontend/mobile-native/src/storage/mod.rs

@@ -15,7 +15,8 @@ use chrono::{DateTime, NaiveDateTime, Utc};
 #[cfg(any(target_os = "android", target_os = "ios", test))]
 use crate::cache::{MobileBlobCache, is_mobile_binary_file_token, should_cache_payload_as_file};
 use crate::{
-  Blob, CrawlResult, DocClock, DocRecord, DocUpdate, ListedBlob, MatchRange, Result, SearchHit, SetBlob, UniffiError,
+  Blob, CrawlResult, DocClock, DocIndexedClock, DocRecord, DocUpdate, IndexAggregateResult, IndexSearchResult,
+  ListedBlob, Result, SetBlob, UniffiError,
   payload_codec::{decode_base64_data, encode_base64_data},
 };