Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-19 11:06:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): import progress & perf (#15197)

Browse Source 
#### PR Dependency Tree


* **PR #15197** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a new import pipeline with “plan then commit” batch handling for
Markdown, Notion HTML, Obsidian, and Bear backups.
* Enabled native import sessions with progress, cancellation, and
batch-by-batch committing (including assets, folders, icons, and tags).
  * Added web preflight limits for ZIP and multi-file imports.
* **Bug Fixes**
* Improved import error/warning reporting and continued processing when
some items fail.
* Strengthened snapshot-based file/directory picking to preserve paths.
* **Chores**
  * Updated project packaging/configuration for the new import workflow.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-06 01:29:24 +08:00
committed by GitHub
parent 477015f064
commit 8d72e4dc29
106 changed files with 3329 additions and 11690 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/native/Cargo.toml
+2 -5
View File
@@ -10,11 +10,8 @@ crate-type = ["cdylib"]
[dependencies]
aes-gcm = { workspace = true }
affine_common = { workspace = true, features = [
"hashcash",
"napi",
"ydoc-loader",
] }
affine_common = { workspace = true, features = ["hashcash", "napi"] }
affine_doc_loader = { workspace = true }
anyhow = { workspace = true }
assetpack-core = "0.1.0"
assetpack-transform-precomp2 = "0.1.0"
packages/backend/native/src/doc.rs
+15 -15
View File
@@ -1,6 +1,6 @@
use affine_common::{
doc_parser::{self, BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent},
napi_utils::map_napi_err,
use affine_common::napi_utils::map_napi_err;
use affine_doc_loader::{
self as doc_loader, BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent,
};
use napi::bindgen_prelude::*;
use napi_derive::napi;
@@ -109,7 +109,7 @@ impl From<CrawlResult> for NativeCrawlResult {
#[napi]
pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result<NativeCrawlResult> {
let result = map_napi_err(
doc_parser::parse_doc_from_binary(doc_bin.into(), doc_id),
doc_loader::parse_doc_from_binary(doc_bin.into(), doc_id),
Status::GenericFailure,
)?;
Ok(result.into())
@@ -118,7 +118,7 @@ pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result<NativeCr
#[napi]
pub fn parse_page_doc(doc_bin: Buffer, max_summary_length: Option<i32>) -> Result<Option<NativePageDocContent>> {
let result = map_napi_err(
doc_parser::parse_page_doc(doc_bin.into(), max_summary_length.map(|v| v as isize)),
doc_loader::parse_page_doc(doc_bin.into(), max_summary_length.map(|v| v as isize)),
Status::GenericFailure,
)?;
Ok(result.map(Into::into))
@@ -126,7 +126,7 @@ pub fn parse_page_doc(doc_bin: Buffer, max_summary_length: Option<i32>) -> Resul
#[napi]
pub fn parse_workspace_doc(doc_bin: Buffer) -> Result<Option<NativeWorkspaceDocContent>> {
let result = map_napi_err(doc_parser::parse_workspace_doc(doc_bin.into()), Status::GenericFailure)?;
let result = map_napi_err(doc_loader::parse_workspace_doc(doc_bin.into()), Status::GenericFailure)?;
Ok(result.map(Into::into))
}
@@ -138,7 +138,7 @@ pub fn parse_doc_to_markdown(
doc_url_prefix: Option<String>,
) -> Result<NativeMarkdownResult> {
let result = map_napi_err(
doc_parser::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false), doc_url_prefix),
doc_loader::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false), doc_url_prefix),
Status::GenericFailure,
)?;
Ok(result.into())
@@ -147,7 +147,7 @@ pub fn parse_doc_to_markdown(
#[napi]
pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option<bool>) -> Result<Vec<String>> {
let result = map_napi_err(
doc_parser::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false)),
doc_loader::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false)),
Status::GenericFailure,
)?;
Ok(result)
@@ -165,7 +165,7 @@ pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option<boo
#[napi]
pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::build_full_doc(&title, &markdown, &doc_id),
doc_loader::build_full_doc(&title, &markdown, &doc_id),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -184,7 +184,7 @@ pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String)
#[napi]
pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, doc_id: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_doc(&existing_binary, &new_markdown, &doc_id),
doc_loader::update_doc(&existing_binary, &new_markdown, &doc_id),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -202,7 +202,7 @@ pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, d
#[napi]
pub fn update_doc_title(existing_binary: Buffer, title: String, doc_id: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_doc_title(&existing_binary, &doc_id, &title),
doc_loader::update_doc_title(&existing_binary, &doc_id, &title),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -229,7 +229,7 @@ pub fn update_doc_properties(
updated_by: Option<String>,
) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_doc_properties(
doc_loader::update_doc_properties(
&existing_binary,
&properties_doc_id,
&target_doc_id,
@@ -254,7 +254,7 @@ pub fn update_doc_properties(
#[napi]
pub fn add_doc_to_root_doc(root_doc_bin: Buffer, doc_id: String, title: Option<String>) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::add_doc_to_root_doc(root_doc_bin.into(), &doc_id, title.as_deref()),
doc_loader::add_doc_to_root_doc(root_doc_bin.into(), &doc_id, title.as_deref()),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -267,7 +267,7 @@ pub fn build_public_root_doc(root_doc_bin: Buffer, doc_metas: Vec<PublicDocMetaI
.map(|meta| (meta.id.as_str(), meta.title.as_deref()))
.collect::<Vec<_>>();
let result = map_napi_err(
doc_parser::build_public_root_doc(&root_doc_bin, &metas),
doc_loader::build_public_root_doc(&root_doc_bin, &metas),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -285,7 +285,7 @@ pub fn build_public_root_doc(root_doc_bin: Buffer, doc_metas: Vec<PublicDocMetaI
#[napi]
pub fn update_root_doc_meta_title(root_doc_bin: Buffer, doc_id: String, title: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title),
doc_loader::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs
+4 -4
View File
@@ -1,4 +1,4 @@
use affine_common::doc_parser;
use affine_doc_loader as doc_loader;
use chrono::{DateTime, Utc};
use sqlx::{FromRow, PgPool};
use y_octo::Doc;
@@ -102,7 +102,7 @@ async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeRes
let Some(root) = load_current_doc(pool, workspace_id, workspace_id).await? else {
return Ok(Vec::new());
};
let ids = doc_parser::get_doc_ids_from_binary(root.blob, false)
let ids = doc_loader::get_doc_ids_from_binary(root.blob, false)
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs root doc parse failed: {err}")))?;
let mut ids = ids;
ids.sort();
@@ -206,7 +206,7 @@ async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_i
}
fn extract_refs(snapshot: &SnapshotRow) -> RuntimeResult<Vec<ExtractedRef>> {
let parsed = doc_parser::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone())
let parsed = doc_loader::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone())
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs parse failed: {err}")))?;
let mut refs = Vec::new();
for block in parsed.blocks {
@@ -232,7 +232,7 @@ mod tests {
fn doc_blob_refs_extracts_image_refs() {
let doc_id = "doc-blob-ref-test".to_string();
let blob =
doc_parser::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build");
doc_loader::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build");
let snapshot = SnapshotRow {
workspace_id: "workspace".to_string(),
doc_id,