feat(server): impl native reader for server (#14100)

packages/backend/native/Cargo.toml

@@ -8,18 +8,22 @@ version      = "1.0.0"
 crate-type = ["cdylib"]
 
 [dependencies]
-affine_common = { workspace = true, features = ["doc-loader", "hashcash"] }
-chrono        = { workspace = true }
-file-format   = { workspace = true }
-infer         = { workspace = true }
-mp4parse      = { workspace = true }
-napi          = { workspace = true, features = ["async"] }
-napi-derive   = { workspace = true }
-rand          = { workspace = true }
-sha3          = { workspace = true }
-tiktoken-rs   = { workspace = true }
-v_htmlescape  = { workspace = true }
-y-octo        = { workspace = true, features = ["large_refs"] }
+affine_common = { workspace = true, features = [
+  "doc-loader",
+  "hashcash",
+  "ydoc-loader",
+] }
+chrono = { workspace = true }
+file-format = { workspace = true }
+infer = { workspace = true }
+mp4parse = { workspace = true }
+napi = { workspace = true, features = ["async"] }
+napi-derive = { workspace = true }
+rand = { workspace = true }
+sha3 = { workspace = true }
+tiktoken-rs = { workspace = true }
+v_htmlescape = { workspace = true }
+y-octo = { workspace = true, features = ["large_refs"] }
 
 [target.'cfg(not(target_os = "linux"))'.dependencies]
 mimalloc = { workspace = true }

packages/backend/native/index.d.ts

@@ -27,6 +27,29 @@ export declare function mergeUpdatesInApplyWay(updates: Array<Buffer>): Buffer
 
 export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
 
+export interface NativeBlockInfo {
+  blockId: string
+  flavour: string
+  content?: Array<string>
+  blob?: Array<string>
+  refDocId?: Array<string>
+  refInfo?: Array<string>
+  parentFlavour?: string
+  parentBlockId?: string
+  additional?: string
+}
+
+export interface NativeCrawlResult {
+  blocks: Array<NativeBlockInfo>
+  title: string
+  summary: string
+}
+
+export interface NativeMarkdownResult {
+  title: string
+  markdown: string
+}
+
 export interface ParsedDoc {
   name: string
   chunks: Array<Chunk>
@@ -34,4 +57,10 @@ export interface ParsedDoc {
 
 export declare function parseDoc(filePath: string, doc: Buffer): Promise<ParsedDoc>
 
+export declare function parseDocFromBinary(docBin: Buffer, docId: string): NativeCrawlResult
+
+export declare function parseDocToMarkdown(docBin: Buffer, docId: string, aiEditable?: boolean | undefined | null): NativeMarkdownResult
+
+export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?: boolean | undefined | null): Array<string>
+
 export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise<boolean>

packages/backend/native/src/doc.rs

@@ -0,0 +1,93 @@
+use affine_common::doc_parser::{self, BlockInfo, CrawlResult, MarkdownResult};
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+
+#[napi(object)]
+pub struct NativeMarkdownResult {
+  pub title: String,
+  pub markdown: String,
+}
+
+impl From<MarkdownResult> for NativeMarkdownResult {
+  fn from(result: MarkdownResult) -> Self {
+    Self {
+      title: result.title,
+      markdown: result.markdown,
+    }
+  }
+}
+
+#[napi(object)]
+pub struct NativeBlockInfo {
+  pub block_id: String,
+  pub flavour: String,
+  pub content: Option<Vec<String>>,
+  pub blob: Option<Vec<String>>,
+  pub ref_doc_id: Option<Vec<String>>,
+  pub ref_info: Option<Vec<String>>,
+  pub parent_flavour: Option<String>,
+  pub parent_block_id: Option<String>,
+  pub additional: Option<String>,
+}
+
+impl From<BlockInfo> for NativeBlockInfo {
+  fn from(info: BlockInfo) -> Self {
+    Self {
+      block_id: info.block_id,
+      flavour: info.flavour,
+      content: info.content,
+      blob: info.blob,
+      ref_doc_id: info.ref_doc_id,
+      ref_info: info.ref_info,
+      parent_flavour: info.parent_flavour,
+      parent_block_id: info.parent_block_id,
+      additional: info.additional,
+    }
+  }
+}
+
+#[napi(object)]
+pub struct NativeCrawlResult {
+  pub blocks: Vec<NativeBlockInfo>,
+  pub title: String,
+  pub summary: String,
+}
+
+impl From<CrawlResult> for NativeCrawlResult {
+  fn from(result: CrawlResult) -> Self {
+    Self {
+      blocks: result.blocks.into_iter().map(Into::into).collect(),
+      title: result.title,
+      summary: result.summary,
+    }
+  }
+}
+
+#[napi]
+pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result<NativeCrawlResult> {
+  let result = doc_parser::parse_doc_from_binary(doc_bin.into(), doc_id)
+    .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
+  Ok(result.into())
+}
+
+#[napi]
+pub fn parse_doc_to_markdown(
+  doc_bin: Buffer,
+  doc_id: String,
+  ai_editable: Option<bool>,
+) -> Result<NativeMarkdownResult> {
+  let result =
+    doc_parser::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false))
+      .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
+  Ok(result.into())
+}
+
+#[napi]
+pub fn read_all_doc_ids_from_root_doc(
+  doc_bin: Buffer,
+  include_trash: Option<bool>,
+) -> Result<Vec<String>> {
+  let result = doc_parser::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false))
+    .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
+  Ok(result)
+}

packages/backend/native/src/lib.rs

@@ -2,6 +2,7 @@
 
 mod utils;
 
+pub mod doc;
 pub mod doc_loader;
 pub mod file_type;
 pub mod hashcash;