feat(native): sync yocto codes (#14243)

#### PR Dependency Tree


* **PR #14243** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Batch management API for coordinated document mutations and change
tracking.
* New document accessors (IDs, state snapshots, change/delete set
queries) and subscriber count.

* **Chores**
  * Upgraded Rust edition across packages to 2024.
  * Repository-wide formatting, stylistic cleanups and test adjustments.

* **Breaking Changes**
* Removed the Node native bindings package and its JS/TS declarations
and tests (no longer published/available).

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/common/native/src/doc_loader/document.rs

@@ -1,6 +1,6 @@
 use std::{
   io::Cursor,
-  panic::{catch_unwind, AssertUnwindSafe},
+  panic::{AssertUnwindSafe, catch_unwind},
   path::PathBuf,
 };
 
@@ -22,9 +22,7 @@ pub struct DocOptions {
 
 impl Default for DocOptions {
   fn default() -> Self {
-    Self {
-      code_threshold: 1000,
-    }
+    Self { code_threshold: 1000 }
   }
 }
 
@@ -39,9 +37,7 @@ impl Doc {
   }
 
   pub fn with_options(file_path: &str, doc: &[u8], options: DocOptions) -> LoaderResult<Self> {
-    if let Some(kind) =
-      infer::get(&doc[..4096.min(doc.len())]).or(infer::get_from_path(file_path).ok().flatten())
-    {
+    if let Some(kind) = infer::get(&doc[..4096.min(doc.len())]).or(infer::get_from_path(file_path).ok().flatten()) {
       if kind.extension() == "pdf" {
         return Self::load_pdf(file_path, doc);
       } else if kind.extension() == "docx" {
@@ -66,11 +62,10 @@ impl Doc {
         }
         "rs" | "c" | "cpp" | "h" | "hpp" | "js" | "ts" | "tsx" | "go" | "py" => {
           let name = path.full_str().to_string();
-          let loader =
-            SourceCodeLoader::from_string(string).with_parser_option(LanguageParserOptions {
-              language: get_language_by_filename(&name)?,
-              parser_threshold: options.code_threshold,
-            });
+          let loader = SourceCodeLoader::from_string(string).with_parser_option(LanguageParserOptions {
+            language: get_language_by_filename(&name)?,
+            parser_threshold: options.code_threshold,
+          });
           let splitter = TokenSplitter::default();
           return Self::from_loader(file_path, loader, splitter);
         }
@@ -89,10 +84,7 @@ impl Doc {
     splitter: impl TextSplitter + 'static,
   ) -> Result<Doc, LoaderError> {
     let name = file_path.to_string();
-    let chunks = catch_unwind(AssertUnwindSafe(|| {
-      Self::get_chunks_from_loader(loader, splitter)
-    }))
-    .map_err(|e| {
+    let chunks = catch_unwind(AssertUnwindSafe(|| Self::get_chunks_from_loader(loader, splitter))).map_err(|e| {
       LoaderError::Other(match e.downcast::<String>() {
         Ok(v) => *v,
         Err(e) => match e.downcast::<&str>() {
@@ -124,8 +116,7 @@ impl Doc {
   }
 
   fn load_docx(file_path: &str, doc: &[u8]) -> LoaderResult<Self> {
-    let loader = DocxLoader::new(Cursor::new(doc))
-      .ok_or(LoaderError::Other("Failed to parse docx document".into()))?;
+    let loader = DocxLoader::new(Cursor::new(doc)).ok_or(LoaderError::Other("Failed to parse docx document".into()))?;
     let splitter = TokenSplitter::default();
     Self::from_loader(file_path, loader, splitter)
   }
@@ -175,8 +166,7 @@ mod tests {
       let buffer = read(fixtures.join(fixture)).unwrap();
       let doc = Doc::with_options(fixture, &buffer, DocOptions { code_threshold: 0 }).unwrap();
       for chunk in doc.chunks.iter() {
-        let output =
-          read_to_string(fixtures.join(format!("{}.{}.md", fixture, chunk.index))).unwrap();
+        let output = read_to_string(fixtures.join(format!("{}.{}.md", fixture, chunk.index))).unwrap();
         assert_eq!(chunk.content, output);
       }
     }

packages/common/native/src/doc_loader/loader/docx.rs

@@ -61,9 +61,7 @@ mod tests {
       for (idx, doc) in documents.into_iter().enumerate() {
         assert_eq!(
           doc.page_content,
-          String::from_utf8_lossy(
-            &read(get_fixtures_path().join(format!("demo.docx.{}.md", idx))).unwrap()
-          )
+          String::from_utf8_lossy(&read(get_fixtures_path().join(format!("demo.docx.{}.md", idx))).unwrap())
         );
       }
     }

packages/common/native/src/doc_loader/loader/html.rs

@@ -29,10 +29,8 @@ impl<R: Read> HtmlLoader<R> {
 impl<R: Read + Send + Sync + 'static> Loader for HtmlLoader<R> {
   fn load(mut self) -> LoaderResult<Vec<Document>> {
     let cleaned_html = readability::extractor::extract(&mut self.html, &self.url)?;
-    let doc =
-      Document::new(format!("{}\n{}", cleaned_html.title, cleaned_html.text)).with_metadata(
-        HashMap::from([("source".to_string(), Value::from(self.url.as_str()))]),
-      );
+    let doc = Document::new(format!("{}\n{}", cleaned_html.title, cleaned_html.text))
+      .with_metadata(HashMap::from([("source".to_string(), Value::from(self.url.as_str()))]));
 
     Ok(vec![doc])
   }
@@ -46,10 +44,7 @@ mod tests {
   fn test_html_loader() {
     let input = "<p>Hello world!</p>";
 
-    let html_loader = HtmlLoader::new(
-      input.as_bytes(),
-      Url::parse("https://example.com/").unwrap(),
-    );
+    let html_loader = HtmlLoader::new(input.as_bytes(), Url::parse("https://example.com/").unwrap());
 
     let documents = html_loader.load().unwrap();
 
@@ -66,16 +61,14 @@ mod tests {
   #[test]
   fn test_html_load_from_path() {
     let buffer = include_bytes!("../../../fixtures/sample.html");
-    let html_loader = HtmlLoader::new(
-      Cursor::new(buffer),
-      Url::parse("https://example.com/").unwrap(),
-    );
+    let html_loader = HtmlLoader::new(Cursor::new(buffer), Url::parse("https://example.com/").unwrap());
 
     let documents = html_loader.load().unwrap();
 
-    let expected = "Example Domain\n\n        This domain is for use in illustrative examples in \
-                    documents. You may\n        use this domain in literature without prior \
-                    coordination or asking for\n        permission.\n      More information...";
+    let expected =
+      "Example Domain\n\n        This domain is for use in illustrative examples in documents. You may\n        use \
+       this domain in literature without prior coordination or asking for\n        permission.\n      More \
+       information...";
 
     assert_eq!(documents.len(), 1);
     assert_eq!(

packages/common/native/src/doc_loader/loader/mod.rs

@@ -23,6 +23,6 @@ pub trait Loader: Send + Sync {
 pub use docx::DocxLoader;
 pub use html::HtmlLoader;
 pub use pdf::PdfExtractLoader;
-pub use source::{get_language_by_filename, LanguageParserOptions, SourceCodeLoader};
+pub use source::{LanguageParserOptions, SourceCodeLoader, get_language_by_filename};
 pub use text::TextLoader;
 pub use url::Url;

packages/common/native/src/doc_loader/loader/pdf.rs

@@ -1,4 +1,4 @@
-use pdf_extract::{output_doc, output_doc_encrypted, PlainTextOutput};
+use pdf_extract::{PlainTextOutput, output_doc, output_doc_encrypted};
 
 /**
  * modified from https://github.com/Abraxas-365/langchain-rust/tree/v4.6.0/src/document_loaders
@@ -72,8 +72,7 @@ mod tests {
     assert_eq!(docs.len(), 1);
     assert_eq!(
       &docs[0].page_content[..100],
-      "\n\nSample PDF\nThis is a simple PDF file. Fun fun fun.\n\nLorem ipsum dolor  sit amet,  \
-       consectetuer  a"
+      "\n\nSample PDF\nThis is a simple PDF file. Fun fun fun.\n\nLorem ipsum dolor  sit amet,  consectetuer  a"
     );
   }
 

packages/common/native/src/doc_loader/loader/source/mod.rs

@@ -3,7 +3,7 @@
  */
 mod parser;
 
-pub use parser::{get_language_by_filename, LanguageParser, LanguageParserOptions};
+pub use parser::{LanguageParser, LanguageParserOptions, get_language_by_filename};
 
 use super::*;
 

packages/common/native/src/doc_loader/loader/source/parser.rs

@@ -59,11 +59,7 @@ pub struct LanguageParser {
 
 impl Debug for LanguageParser {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-    write!(
-      f,
-      "LanguageParser {{ language: {:?} }}",
-      self.parser_options.language
-    )
+    write!(f, "LanguageParser {{ language: {:?} }}", self.parser_options.language)
   }
 }
 
@@ -77,10 +73,7 @@ impl Clone for LanguageParser {
 }
 
 pub fn get_language_by_filename(name: &str) -> LoaderResult<Language> {
-  let extension = name
-    .split('.')
-    .next_back()
-    .ok_or(LoaderError::UnsupportedLanguage)?;
+  let extension = name.split('.').next_back().ok_or(LoaderError::UnsupportedLanguage)?;
   let language = match extension.to_lowercase().as_str() {
     "rs" => Language::Rust,
     "c" => Language::C,
@@ -133,10 +126,7 @@ impl LanguageParser {
 
 impl LanguageParser {
   pub fn parse_code(&mut self, code: &String) -> LoaderResult<Vec<Document>> {
-    let tree = self
-      .parser
-      .parse(code, None)
-      .ok_or(LoaderError::UnsupportedLanguage)?;
+    let tree = self.parser.parse(code, None).ok_or(LoaderError::UnsupportedLanguage)?;
     if self.parser_options.parser_threshold > tree.root_node().end_position().row as u64 {
       return Ok(vec![Document::new(code).with_metadata(HashMap::from([
         (
@@ -152,11 +142,7 @@ impl LanguageParser {
     self.extract_functions_classes(tree, code)
   }
 
-  pub fn extract_functions_classes(
-    &self,
-    tree: Tree,
-    code: &String,
-  ) -> LoaderResult<Vec<Document>> {
+  pub fn extract_functions_classes(&self, tree: Tree, code: &String) -> LoaderResult<Vec<Document>> {
     let mut chunks = Vec::new();
 
     let count = tree.root_node().child_count();

packages/common/native/src/doc_loader/loader/text.rs

@@ -10,9 +10,7 @@ pub struct TextLoader {
 
 impl TextLoader {
   pub fn new<T: Into<String>>(input: T) -> Self {
-    Self {
-      content: input.into(),
-    }
+    Self { content: input.into() }
   }
 }
 

packages/common/native/src/doc_loader/mod.rs

@@ -7,8 +7,8 @@ mod types;
 pub use document::{Chunk, Doc};
 pub use error::{LoaderError, LoaderResult};
 use loader::{
-  get_language_by_filename, DocxLoader, HtmlLoader, LanguageParserOptions, Loader,
-  PdfExtractLoader, SourceCodeLoader, TextLoader, Url,
+  DocxLoader, HtmlLoader, LanguageParserOptions, Loader, PdfExtractLoader, SourceCodeLoader, TextLoader, Url,
+  get_language_by_filename,
 };
 use splitter::{MarkdownSplitter, TextSplitter, TextSplitterError, TokenSplitter};
 use types::Document;

packages/common/native/src/doc_loader/splitter/options.rs

@@ -2,7 +2,7 @@
  * modified from https://github.com/Abraxas-365/langchain-rust/tree/v4.6.0/src/text_splitter
  */
 use text_splitter::ChunkConfig;
-use tiktoken_rs::{get_bpe_from_model, get_bpe_from_tokenizer, tokenizer::Tokenizer, CoreBPE};
+use tiktoken_rs::{CoreBPE, get_bpe_from_model, get_bpe_from_tokenizer, tokenizer::Tokenizer};
 
 use super::TextSplitterError;
 
@@ -79,8 +79,8 @@ impl TryFrom<&SplitterOptions> for ChunkConfig<CoreBPE> {
 
   fn try_from(options: &SplitterOptions) -> Result<Self, Self::Error> {
     let tk = if !options.encoding_name.is_empty() {
-      let tokenizer = SplitterOptions::get_tokenizer_from_str(&options.encoding_name)
-        .ok_or(TextSplitterError::TokenizerNotFound)?;
+      let tokenizer =
+        SplitterOptions::get_tokenizer_from_str(&options.encoding_name).ok_or(TextSplitterError::TokenizerNotFound)?;
 
       get_bpe_from_tokenizer(tokenizer).map_err(|_| TextSplitterError::InvalidTokenizer)?
     } else {