feat(server): adapt doc loader for server native (#9942)

packages/common/native/src/doc_loader/loader/docx.rs

@@ -24,7 +24,7 @@ impl DocxLoader {
 }
 
 impl Loader for DocxLoader {
-  fn load(self) -> Result<Vec<Document>, LoaderError> {
+  fn load(self) -> LoaderResult<Vec<Document>> {
     let doc = self.extract_text_to_doc();
     Ok(vec![doc])
   }

packages/common/native/src/doc_loader/loader/error.rs

@@ -1,42 +0,0 @@
-use std::{io, str::Utf8Error, string::FromUtf8Error};
-
-use thiserror::Error;
-
-/**
- * modified from https://github.com/Abraxas-365/langchain-rust/tree/v4.6.0/src/document_loaders
- */
-use super::*;
-
-#[derive(Error, Debug)]
-pub enum LoaderError {
-  #[error("{0}")]
-  TextSplitterError(#[from] TextSplitterError),
-
-  #[error(transparent)]
-  IOError(#[from] io::Error),
-
-  #[error(transparent)]
-  Utf8Error(#[from] Utf8Error),
-
-  #[error(transparent)]
-  FromUtf8Error(#[from] FromUtf8Error),
-
-  #[cfg(feature = "pdf-extract")]
-  #[error(transparent)]
-  PdfExtractError(#[from] pdf_extract::Error),
-
-  #[cfg(feature = "pdf-extract")]
-  #[error(transparent)]
-  PdfExtractOutputError(#[from] pdf_extract::OutputError),
-
-  #[error(transparent)]
-  ReadabilityError(#[from] readability::error::Error),
-
-  #[error("Unsupported source language")]
-  UnsupportedLanguage,
-
-  #[error("Error: {0}")]
-  OtherError(String),
-}
-
-pub type LoaderResult<T> = Result<T, LoaderError>;

packages/common/native/src/doc_loader/loader/html.rs

@@ -27,7 +27,7 @@ impl<R: Read> HtmlLoader<R> {
 }
 
 impl<R: Read + Send + Sync + 'static> Loader for HtmlLoader<R> {
-  fn load(mut self) -> Result<Vec<Document>, LoaderError> {
+  fn load(mut self) -> LoaderResult<Vec<Document>> {
     let cleaned_html = readability::extractor::extract(&mut self.html, &self.url)?;
     let doc =
       Document::new(format!("{}\n{}", cleaned_html.title, cleaned_html.text)).with_metadata(

packages/common/native/src/doc_loader/loader/mod.rs

@@ -1,5 +1,4 @@
 mod docx;
-mod error;
 mod html;
 mod pdf;
 mod source;
@@ -11,11 +10,8 @@ use super::*;
 
 // modified from https://github.com/Abraxas-365/langchain-rust/tree/v4.6.0/src/document_loaders
 pub trait Loader: Send + Sync {
-  fn load(self) -> Result<Vec<Document>, LoaderError>;
-  fn load_and_split<TS: TextSplitter + 'static>(
-    self,
-    splitter: TS,
-  ) -> Result<Vec<Document>, LoaderError>
+  fn load(self) -> LoaderResult<Vec<Document>>;
+  fn load_and_split<TS: TextSplitter + 'static>(self, splitter: TS) -> LoaderResult<Vec<Document>>
   where
     Self: Sized,
   {
@@ -25,7 +21,6 @@ pub trait Loader: Send + Sync {
 }
 
 pub use docx::DocxLoader;
-pub use error::{LoaderError, LoaderResult};
 pub use html::HtmlLoader;
 pub use pdf::PdfExtractLoader;
 pub use source::{get_language_by_filename, LanguageParserOptions, SourceCodeLoader};

packages/common/native/src/doc_loader/loader/pdf.rs

@@ -12,8 +12,7 @@ pub struct PdfExtractLoader {
 
 impl PdfExtractLoader {
   pub fn new<R: Read>(reader: R) -> Result<Self, LoaderError> {
-    let document = pdf_extract::Document::load_from(reader)
-      .map_err(|e| LoaderError::OtherError(e.to_string()))?;
+    let document = pdf_extract::Document::load_from(reader)?;
     Ok(Self { document })
   }
 }
@@ -38,7 +37,7 @@ impl PdfExtractLoader {
 }
 
 impl Loader for PdfExtractLoader {
-  fn load(self) -> Result<Vec<Document>, LoaderError> {
+  fn load(self) -> LoaderResult<Vec<Document>> {
     let doc = self.extract_text_to_doc()?;
     Ok(vec![doc])
   }

packages/common/native/src/doc_loader/loader/source/mod.rs

@@ -30,7 +30,7 @@ impl SourceCodeLoader {
 }
 
 impl Loader for SourceCodeLoader {
-  fn load(self) -> Result<Vec<Document>, LoaderError> {
+  fn load(self) -> LoaderResult<Vec<Document>> {
     let options = self.parser_option.clone();
 
     let docs = LanguageParser::from_language(options.language)

packages/common/native/src/doc_loader/loader/text.rs

@@ -17,7 +17,7 @@ impl TextLoader {
 }
 
 impl Loader for TextLoader {
-  fn load(self) -> Result<Vec<Document>, LoaderError> {
+  fn load(self) -> LoaderResult<Vec<Document>> {
     let doc = Document::new(self.content);
     Ok(vec![doc])
   }