feat: introduce fuzzy search for native indexer (#14109)

packages/common/native/Cargo.toml

@@ -17,10 +17,11 @@ doc-loader = [
   "strum_macros",
   "text-splitter",
   "thiserror",
+  "tiktoken-rs",
   "tree-sitter",
   "url",
 ]
-hashcash = ["sha3", "rand"]
+hashcash = ["chrono", "sha3", "rand"]
 tree-sitter = [
   "cc",
   "dep:tree-sitter",
@@ -39,24 +40,24 @@ tree-sitter = [
 ydoc-loader = ["assert-json-diff", "serde", "serde_json", "thiserror", "y-octo"]
 
 [dependencies]
-chrono = { workspace = true }
-rand   = { workspace = true, optional = true }
-sha3   = { workspace = true, optional = true }
-
 assert-json-diff = { workspace = true, optional = true }
+chrono = { workspace = true, optional = true }
 docx-parser = { workspace = true, optional = true }
 infer = { workspace = true, optional = true }
 path-ext = { workspace = true, optional = true }
 pdf-extract = { workspace = true, optional = true }
+rand = { workspace = true, optional = true }
 readability = { workspace = true, optional = true, default-features = false }
 serde = { workspace = true, optional = true, features = ["derive"] }
 serde_json = { workspace = true, optional = true }
+sha3 = { workspace = true, optional = true }
 strum_macros = { workspace = true, optional = true }
 text-splitter = { workspace = true, features = [
   "markdown",
   "tiktoken-rs",
 ], optional = true }
 thiserror = { workspace = true, optional = true }
+tiktoken-rs = { workspace = true, optional = true }
 tree-sitter = { workspace = true, optional = true }
 tree-sitter-c = { workspace = true, optional = true }
 tree-sitter-c-sharp = { workspace = true, optional = true }
@@ -72,11 +73,10 @@ tree-sitter-typescript = { workspace = true, optional = true }
 url = { workspace = true, optional = true }
 y-octo = { workspace = true, optional = true }
 
-tiktoken-rs = { workspace = true }
-
 [dev-dependencies]
-criterion2 = { workspace = true }
-rayon      = { workspace = true }
+criterion = { workspace = true }
+rayon     = { workspace = true }
+tempfile  = "3"
 
 [build-dependencies]
 cc = { version = "1", optional = true }

packages/common/nbstore/src/impls/sqlite/db.ts

@@ -99,7 +99,7 @@ export interface NativeDBApis {
     id: string,
     indexName: string,
     query: string
-  ) => Promise<{ id: string; score: number }[]>;
+  ) => Promise<{ id: string; score: number; terms: Array<string> }[]>;
   ftsGetDocument: (
     id: string,
     indexName: string,

packages/common/y-octo/core/src/doc/codec/refs.rs

@@ -254,6 +254,11 @@ impl Node {
         let mut ritem = unsafe { rref.get_mut_unchecked() };
         let llen = litem.len();
 
+        let parent_kind = match &litem.parent {
+          Some(Parent::Type(ty)) => ty.ty().map(|ty| ty.kind()),
+          _ => None,
+        };
+
         if litem.id.client != ritem.id.client
                     // not same delete status
                     || litem.deleted() != ritem.deleted()
@@ -277,6 +282,13 @@ impl Node {
             l.extend(r.drain(0..));
           }
           (Content::String(l), Content::String(r)) => {
+            let allow_merge_string =
+              matches!(parent_kind, Some(YTypeKind::Text | YTypeKind::XMLText));
+
+            if !allow_merge_string {
+              return false;
+            }
+
             *l += r;
           }
           (Content::Any(l), Content::Any(r)) => {

packages/common/y-octo/core/src/doc/types/array.rs

@@ -4,21 +4,50 @@ impl_type!(Array);
 
 impl ListType for Array {}
 
-pub struct ArrayIter<'a>(ListIterator<'a>);
+pub struct ArrayIter<'a> {
+  iter: ListIterator<'a>,
+  pending: Option<PendingArrayValues>,
+}
+
+enum PendingArrayValues {
+  Any { values: Vec<Any>, index: usize },
+}
 
 impl Iterator for ArrayIter<'_> {
   type Item = Value;
 
   fn next(&mut self) -> Option<Self::Item> {
-    for item in self.0.by_ref() {
+    loop {
+      if let Some(PendingArrayValues::Any { values, index }) = &mut self.pending {
+        if *index < values.len() {
+          let value = values[*index].clone();
+          *index += 1;
+          return Some(Value::Any(value));
+        }
+        self.pending = None;
+      }
+
+      let item = self.iter.next()?;
       if let Some(item) = item.get() {
-        if item.countable() {
-          return Some(Value::from(&item.content));
+        if !item.countable() {
+          continue;
+        }
+
+        match &item.content {
+          Content::Any(values) if !values.is_empty() => {
+            if values.len() > 1 {
+              self.pending = Some(PendingArrayValues::Any {
+                values: values.clone(),
+                index: 1,
+              });
+            }
+
+            return Some(Value::Any(values[0].clone()));
+          }
+          _ => return Some(Value::from(&item.content)),
         }
       }
     }
-
-    None
   }
 }
 
@@ -46,7 +75,10 @@ impl Array {
   }
 
   pub fn iter(&self) -> ArrayIter<'_> {
-    ArrayIter(self.iter_item())
+    ArrayIter {
+      iter: self.iter_item(),
+      pending: None,
+    }
   }
 
   pub fn push<V: Into<Value>>(&mut self, val: V) -> JwstCodecResult {

packages/common/y-octo/node/src/utils.rs

@@ -78,7 +78,6 @@ pub fn get_any_from_js_object(object: Object) -> Result<Any> {
           })
         }) {
           if let Ok(value) = object.get_named_property_unchecked::<Unknown>(&key) {
-            println!("key: {}", key);
             map.insert(key, get_any_from_js_unknown(value)?);
           }
         }

packages/common/y-octo/utils/bin/bench_result_render.rs

@@ -85,7 +85,7 @@ fn convert_to_markdown() -> impl Iterator<Item = String> {
         let (changes_dur_secs, changes_err_secs) = process_duration(&changes_duration)?;
 
         let diff = -(1.0 - changes_dur_secs / base_dur_secs) * 100.0;
-        difference = format!("{:+.2}%", diff);
+        difference = format!("{diff:+.2}%");
 
         if is_significant(
           changes_dur_secs,
@@ -93,7 +93,7 @@ fn convert_to_markdown() -> impl Iterator<Item = String> {
           base_dur_secs,
           base_err_secs,
         ) {
-          difference = format!("**{}**", difference);
+          difference = format!("**{difference}**");
         }
       }
 
@@ -128,7 +128,7 @@ fn main() {
   ];
 
   for line in headers.into_iter().chain(convert_to_markdown()) {
-    println!("{}", line);
+    println!("{line}");
   }
   println!("</details>");
 }

packages/common/y-octo/utils/bin/doc_merger.rs

@@ -35,7 +35,7 @@ fn load_path(path: &str) -> Result<Vec<Vec<u8>>, Error> {
     paths.sort();
 
     for path in paths {
-      println!("read {:?}", path);
+      println!("read {path:?}");
       updates.push(read(path)?);
     }
     Ok(updates)
@@ -66,7 +66,7 @@ fn jwst_merge(path: &str) {
   let history = doc.history().parse_store(Default::default());
   println!("history: {:?}", ts.elapsed());
   for history in history.iter().take(100) {
-    println!("history: {:?}", history);
+    println!("history: {history:?}");
   }
 
   doc.gc().unwrap();