From 9581432d217205422c104f1d5c5e95925b229209 Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:20:52 +0800 Subject: [PATCH] feat(core): onenote importer (#15198) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #### PR Dependency Tree * **PR #15198** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) ## Summary by CodeRabbit * **New Features** * Added OneNote import support for `.one`, `.onetoc2`, and `.onepkg`, including OneNote-to-markdown content conversion. * OneNote now appears in the import flow with a dedicated label and format tooltip, and file pickers recognize the new OneNote type. * **Bug Fixes** * Import options that are desktop-only are now disabled when not running in the desktop app, with clear messaging. * Improved imported asset handling by converting non-image embedded assets into attachments for more consistent results. --- Cargo.lock | 256 +++++++-- Cargo.toml | 10 +- .../affine/shared/src/utils/file/filesys.ts | 7 + .../server/src/__tests__/models/user.spec.ts | 37 +- .../src/core/mail/__tests__/job.spec.ts | 27 + packages/backend/server/src/core/mail/job.ts | 36 +- packages/backend/server/src/models/user.ts | 4 + .../apps/electron/src/shared/import.ts | 3 +- .../core/src/desktop/dialogs/import/index.tsx | 79 ++- .../src/desktop/dialogs/import/styles.css.ts | 13 + .../src/modules/import/services/service.ts | 10 + .../i18n/src/i18n-completenesses.json | 6 +- packages/frontend/i18n/src/i18n.gen.ts | 12 + packages/frontend/i18n/src/resources/en.json | 3 + .../frontend/i18n/src/resources/zh-Hans.json | 3 + packages/frontend/native/Cargo.toml | 6 + packages/frontend/native/src/import/mod.rs | 5 + .../native/src/import/onenote/content/mod.rs | 534 ++++++++++++++++++ .../src/import/onenote/content/rich_text.rs | 171 ++++++ .../frontend/native/src/import/onenote/mod.rs | 444 +++++++++++++++ .../{import_session.rs => import/session.rs} | 63 ++- packages/frontend/native/src/lib.rs | 4 +- 22 files changed, 1621 insertions(+), 112 deletions(-) create mode 100644 packages/frontend/native/src/import/mod.rs create mode 100644 packages/frontend/native/src/import/onenote/content/mod.rs create mode 100644 packages/frontend/native/src/import/onenote/content/rich_text.rs create mode 100644 packages/frontend/native/src/import/onenote/mod.rs rename packages/frontend/native/src/{import_session.rs => import/session.rs} (84%) diff --git a/Cargo.lock b/Cargo.lock index 605bada7b9..ffa99c5394 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,11 +76,11 @@ dependencies = [ [[package]] name = "affine_doc_loader" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3f18f85e1c8528321903d602ca32a5603f4539ca1412c765f0dbaf6aa63f" +checksum = "6af9fa0b8359900e4c150b717e837eec813f29ee6981fa72d83e671974c0f78f" dependencies = [ - "nanoid", + "nanoid 0.5.0", "pulldown-cmark", "serde", "serde_json", @@ -90,13 +90,13 @@ dependencies = [ [[package]] name = "affine_importer" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1119144030a9ad25dee926d333dcb20433ab72f57425cbd159aa57eb9ee35119" +checksum = "0a698069c4f0ae26bbc3c07deaf84bb0d55b0ed80dae44d6879ca1bf63eee8e9" dependencies = [ "affine_doc_loader", "chrono", - "nanoid", + "nanoid 0.5.0", "serde", "serde_json", "sha2 0.11.0", @@ -144,7 +144,7 @@ dependencies = [ "base64-simd", "chrono", "homedir", - "lru", + "lru 0.16.3", "mermaid-rs-renderer", "objc2", "objc2-foundation", @@ -161,21 +161,27 @@ name = "affine_native" version = "0.0.0" dependencies = [ "affine_common", + "affine_doc_loader", "affine_importer", "affine_media_capture", "affine_nbstore", "affine_sqlite_v1", "chrono", + "hex", + "infer", "mermaid-rs-renderer", "mimalloc", "napi", "napi-build", "napi-derive", "once_cell", + "onenote_parser", "serde_json", + "sha2 0.11.0", "sqlx", "thiserror 2.0.18", "tokio", + "typed-path", "typst", "typst-as-lib", "typst-svg", @@ -262,7 +268,7 @@ dependencies = [ "sqlx", "tempfile", "thiserror 2.0.18", - "tiktoken-rs", + "tiktoken-rs 0.7.0", "tokio", "url", "uuid", @@ -695,7 +701,7 @@ checksum = "53d0c374feba1b9a59042a7c1cf00ce7c34b977b9134fe7c42b08e5183729f66" dependencies = [ "paste", "roman-numerals-rs", - "strum", + "strum 0.27.2", "unic-langid", "unicode-normalization", "unscanny", @@ -1003,6 +1009,18 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cab" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171228650e6721d5acc0868a462cd864f49ac5f64e4a42cde270406e64e404d2" +dependencies = [ + "byteorder", + "flate2", + "lzxd", + "time", +] + [[package]] name = "cabac" version = "0.15.0" @@ -1999,9 +2017,9 @@ dependencies = [ [[package]] name = "doc_extractor" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a05485e5262ab131ff0137a369a13cd2e3c5dd2e5c1e57a5c387998cc6fd182f" +checksum = "f43cff8b35cc5673becb65d2417917f540a3e544655071f891d8c564cd48d1c2" dependencies = [ "infer", "office_oxide", @@ -2010,10 +2028,10 @@ dependencies = [ "readability", "serde", "serde_json", - "strum_macros", + "strum_macros 0.28.0", "text-splitter", "thiserror 2.0.18", - "tiktoken-rs", + "tiktoken-rs 0.12.0", "tree-sitter", "tree-sitter-c", "tree-sitter-cpp", @@ -2083,9 +2101,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -2160,6 +2178,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "enum-primitive-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7795da175654fe16979af73f81f26a8ea27638d8d9823d317016888a63dc4c" +dependencies = [ + "num-traits", + "quote", + "syn 2.0.117", +] + [[package]] name = "env_filter" version = "1.0.1" @@ -2196,7 +2225,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2860,6 +2889,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashlink" version = "0.10.0" @@ -3244,7 +3284,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.61.2", ] [[package]] @@ -3745,7 +3785,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4299,6 +4339,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4314,6 +4363,12 @@ dependencies = [ "sha2 0.11.0", ] +[[package]] +name = "lzxd" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17f346186eccb574ba5581acefc514f0c70a642db4f96e245034a0a158a7168" + [[package]] name = "mac" version = "0.1.1" @@ -4405,9 +4460,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -4560,6 +4615,15 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "nanoid" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628de41fe064cc3f0cf07f3d299ee3e73521adaff72278731d5c8cae3797873" +dependencies = [ + "rand 0.9.4", +] + [[package]] name = "napi" version = "3.8.3" @@ -4930,16 +4994,16 @@ dependencies = [ [[package]] name = "office_oxide" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866a8fa5e1a756f8cc9f9f614031ad7e33adc7f6972eab74a35b0c799fe08da5" +checksum = "8900703278aea7d2329f5c490b62653306b1083efc7993ee18e3e86dcfd5afeb" dependencies = [ "atoi_simd", "encoding_rs", "fast-float2", "libc", "log", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "serde", "serde_json", "thiserror 2.0.18", @@ -4971,6 +5035,38 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onenote_parser" +version = "1.1.1" +source = "git+https://github.com/msiemens/onenote.rs?rev=5138a39a3f4e72b840932f9872fecde52fa9da60#5138a39a3f4e72b840932f9872fecde52fa9da60" +dependencies = [ + "bytes", + "cab", + "encoding_rs", + "enum-primitive-derive", + "itertools 0.14.0", + "log", + "lru 0.18.0", + "num-traits", + "onenote_parser-macros", + "sanitize-filename", + "thiserror 2.0.18", + "time", + "typed-path", + "uuid", + "widestring", +] + +[[package]] +name = "onenote_parser-macros" +version = "1.1.1" +source = "git+https://github.com/msiemens/onenote.rs?rev=5138a39a3f4e72b840932f9872fecde52fa9da60#5138a39a3f4e72b840932f9872fecde52fa9da60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "oorandom" version = "11.1.5" @@ -5707,6 +5803,15 @@ name = "quick-xml" version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5765,7 +5870,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6207,7 +6312,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6266,7 +6371,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6370,6 +6475,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sanitize-filename" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" +dependencies = [ + "regex", +] + [[package]] name = "schannel" version = "0.1.29" @@ -7018,6 +7132,7 @@ dependencies = [ "cfg-if", "libc", "psm", + "windows-sys 0.52.0", "windows-sys 0.59.0", ] @@ -7096,7 +7211,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -7111,6 +7235,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subsetter" version = "0.2.6" @@ -7447,7 +7583,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7463,9 +7599,9 @@ dependencies = [ [[package]] name = "text-splitter" -version = "0.27.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7e97f5863248f7d07896a1816bd4110cb1b0b122741f157d702121a270bf33" +checksum = "3f3eeec76988617ff1434d754d7e8e197be2cb8981ea77c17e32f3f7a2c6f95e" dependencies = [ "ahash", "auto_enums", @@ -7475,9 +7611,9 @@ dependencies = [ "itertools 0.14.0", "memchr", "pulldown-cmark", - "strum", + "strum 0.28.0", "thiserror 2.0.18", - "tiktoken-rs", + "tiktoken-rs 0.12.0", ] [[package]] @@ -7587,6 +7723,21 @@ dependencies = [ "rustc-hash 1.1.0", ] +[[package]] +name = "tiktoken-rs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2" +dependencies = [ + "anyhow", + "base64", + "bstr", + "fancy-regex 0.17.0", + "lazy_static", + "regex", + "rustc-hash 2.1.1", +] + [[package]] name = "time" version = "0.3.47" @@ -7938,9 +8089,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.25.10" +version = "0.26.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +checksum = "3c343ed63e3f5c64d1acdecb5d2c13d4e169cb5fde0052106ebaa6c6f27f9e55" dependencies = [ "cc", "regex", @@ -7952,9 +8103,9 @@ dependencies = [ [[package]] name = "tree-sitter-c" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3aad8f0129083a59fe8596157552d2bb7148c492d44c21558d68ca1c722707" +checksum = "a9b2eb57a55fed6b00812912e730b7a275cf4fe98bfd6a5d76263d4438371728" dependencies = [ "cc", "tree-sitter-language", @@ -7972,9 +8123,9 @@ dependencies = [ [[package]] name = "tree-sitter-go" -version = "0.23.4" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13d476345220dbe600147dd444165c5791bf85ef53e28acbedd46112ee18431" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" dependencies = [ "cc", "tree-sitter-language", @@ -7982,9 +8133,9 @@ dependencies = [ [[package]] name = "tree-sitter-javascript" -version = "0.23.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf40bf599e0416c16c125c3cec10ee5ddc7d1bb8b0c60fa5c4de249ad34dc1b1" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" dependencies = [ "cc", "tree-sitter-language", @@ -7998,9 +8149,9 @@ checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" [[package]] name = "tree-sitter-python" -version = "0.23.6" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" dependencies = [ "cc", "tree-sitter-language", @@ -8008,9 +8159,9 @@ dependencies = [ [[package]] name = "tree-sitter-rust" -version = "0.24.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9b18034c684a2420722be8b2a91c9c44f2546b631c039edf575ccba8c61be1" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" dependencies = [ "cc", "tree-sitter-language", @@ -9149,7 +9300,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -9213,19 +9364,6 @@ dependencies = [ "windows-strings 0.4.2", ] -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - [[package]] name = "windows-future" version = "0.2.1" @@ -9745,7 +9883,7 @@ dependencies = [ "byteorder", "log", "loom", - "nanoid", + "nanoid 0.4.0", "nom 8.0.0", "ordered-float", "rand 0.9.4", diff --git a/Cargo.toml b/Cargo.toml index b0f034620b..000420ccaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ resolver = "3" [workspace.dependencies] aes-gcm = "0.10" affine_common = { path = "./packages/common/native" } - affine_doc_loader = "0.1.0" - affine_importer = "0.1.0" + affine_doc_loader = "0.1.2" + affine_importer = "0.1.1" affine_nbstore = { path = "./packages/frontend/native/nbstore" } anyhow = "1" assert-json-diff = "2.0" @@ -32,7 +32,7 @@ resolver = "3" criterion2 = { version = "3", default-features = false } crossbeam-channel = "0.5" dispatch2 = "0.3" - doc_extractor = "0.1.0" + doc_extractor = "0.1.1" dotenvy = "0.15" file-format = { version = "0.28", features = ["reader"] } hex = "0.4" @@ -69,6 +69,9 @@ resolver = "3" objc2-foundation = "0.3" ogg = "0.9" once_cell = "1" + onenote_parser = { git = "https://github.com/msiemens/onenote.rs", rev = "5138a39a3f4e72b840932f9872fecde52fa9da60", package = "onenote_parser", features = [ + "onepkg", + ] } p256 = { version = "0.13", features = ["ecdsa", "pem"] } pulldown-cmark = "0.13" rand = "0.9" @@ -92,6 +95,7 @@ resolver = "3" thiserror = "2" tiktoken-rs = "0.7" tokio = "1.45" + typed-path = "0.12.3" typst = "0.14.2" typst-as-lib = { version = "0.15.4", default-features = false, features = [ "packages", diff --git a/blocksuite/affine/shared/src/utils/file/filesys.ts b/blocksuite/affine/shared/src/utils/file/filesys.ts index f516a84413..b1a025a238 100644 --- a/blocksuite/affine/shared/src/utils/file/filesys.ts +++ b/blocksuite/affine/shared/src/utils/file/filesys.ts @@ -120,6 +120,12 @@ const FileTypes: NonNullable = [ ['.docx'], }, }, + { + description: 'OneNote', + accept: { + 'application/onenote': ['.one', '.onetoc2', '.onepkg'], + }, + }, { description: 'MindMap', accept: { @@ -140,6 +146,7 @@ type AcceptTypes = | 'Html' | 'Zip' | 'Docx' + | 'OneNote' | 'MindMap'; type OpenFileOptions = { diff --git a/packages/backend/server/src/__tests__/models/user.spec.ts b/packages/backend/server/src/__tests__/models/user.spec.ts index 1f8707d5d6..c126db0fe1 100644 --- a/packages/backend/server/src/__tests__/models/user.spec.ts +++ b/packages/backend/server/src/__tests__/models/user.spec.ts @@ -1,4 +1,5 @@ -import ava, { TestFn } from 'ava'; +import { PrismaClient } from '@prisma/client'; +import ava, { type TestFn } from 'ava'; import Sinon from 'sinon'; import { EmailAlreadyUsed, EventBus } from '../../base'; @@ -9,6 +10,7 @@ import { createTestingModule, sleep, type TestingModule } from '../utils'; interface Context { module: TestingModule; models: Models; + db: PrismaClient; user: UserModel; } @@ -19,6 +21,7 @@ test.before(async t => { t.context.user = module.get(UserModel); t.context.models = module.get(Models); + t.context.db = module.get(PrismaClient); t.context.module = module; }); @@ -262,6 +265,38 @@ test('should delete user', async t => { t.is(user2, null); }); +test('should delete user with pending invitation missing normalized email', async t => { + const owner = await t.context.user.create({ + email: 'owner@affine.pro', + }); + const invitee = await t.context.user.create({ + email: 'invitee@affine.pro', + registered: false, + }); + const workspace = await t.context.models.workspace.create(owner.id); + + const invitation = await t.context.db.workspaceInvitation.create({ + data: { + workspaceId: workspace.id, + inviteeUserId: invitee.id, + inviterUserId: owner.id, + requestedRole: 'member', + status: 'pending', + kind: 'email', + }, + }); + + await t.context.user.delete(invitee.id); + + t.is( + await t.context.db.workspaceInvitation.findUnique({ + where: { id: invitation.id }, + }), + null + ); + t.is(await t.context.user.get(invitee.id), null); +}); + test('should trigger user.deleted event', async t => { const event = t.context.module.get(EventBus); const spy = Sinon.spy(); diff --git a/packages/backend/server/src/core/mail/__tests__/job.spec.ts b/packages/backend/server/src/core/mail/__tests__/job.spec.ts index 9e2283f1b7..5bad0da05d 100644 --- a/packages/backend/server/src/core/mail/__tests__/job.spec.ts +++ b/packages/backend/server/src/core/mail/__tests__/job.spec.ts @@ -38,6 +38,7 @@ test('should clear pending mail records when user is deleted', async t => { const userKey = `${sendMailKey}:SignIn:${user.email}`; const userRetryKey = `${sendMailKey}:VerifyEmail:${user.email}`; const anotherKey = `${sendMailKey}:SignIn:${another.email}`; + const senderRetryKey = `${retryMailKey}:MemberInvitation:invited@affine.pro`; await cache.mapSet(sendMailKey, userKey, 1); await cache.mapSet(sendMailKey, anotherKey, 1); @@ -51,6 +52,20 @@ test('should clear pending mail records when user is deleted', async t => { props: { url: 'https://affine.pro/verify' }, }) ); + await cache.mapSet( + retryMailKey, + senderRetryKey, + JSON.stringify({ + startTime: Date.now(), + name: 'MemberInvitation', + to: 'invited@affine.pro', + props: { + user: { $$userId: user.id }, + workspace: { $$workspaceId: 'workspace-id' }, + url: 'https://affine.pro/invite', + }, + }) + ); await mailJob.onUserDeleted({ ...user, ownedWorkspaces: [] }); @@ -62,6 +77,17 @@ test('should clear pending mail records when user is deleted', async t => { to: user.email, } as Jobs['notification.sendMail']) ); + t.true( + await shouldRemove({ + name: 'MemberInvitation', + to: 'invited@affine.pro', + props: { + user: { $$userId: user.id }, + workspace: { $$workspaceId: 'workspace-id' }, + url: 'https://affine.pro/invite', + }, + } as Jobs['notification.sendMail']) + ); t.false( await shouldRemove({ to: another.email, @@ -69,6 +95,7 @@ test('should clear pending mail records when user is deleted', async t => { ); t.is(await cache.mapGet(sendMailKey, userKey), undefined); t.is(await cache.mapGet(retryMailKey, userRetryKey), undefined); + t.is(await cache.mapGet(retryMailKey, senderRetryKey), undefined); t.is(await cache.mapGet(sendMailKey, anotherKey), 1); }); diff --git a/packages/backend/server/src/core/mail/job.ts b/packages/backend/server/src/core/mail/job.ts index 92dd2e5377..4b21d00f75 100644 --- a/packages/backend/server/src/core/mail/job.ts +++ b/packages/backend/server/src/core/mail/job.ts @@ -126,6 +126,38 @@ export class MailJob { ); } + private isInvitationMailSentByUser( + job: Jobs['notification.sendMail'], + userId: string + ) { + return ( + job.name === 'MemberInvitation' && + 'user' in job.props && + typeof job.props.user === 'object' && + job.props.user !== null && + '$$userId' in job.props.user && + job.props.user.$$userId === userId + ); + } + + private async deleteInvitationMailCacheBySender(userId: string) { + const keys = await this.cache.mapKeys(retryMailKey); + await Promise.all( + keys.map(async key => { + const job = await this.cache.mapGet< + Jobs['notification.sendMail'] | string + >(retryMailKey, key); + const jobData = + typeof job === 'string' + ? (JSON.parse(job) as Jobs['notification.sendMail']) + : job; + if (jobData && this.isInvitationMailSentByUser(jobData, userId)) { + await this.cache.mapDelete(retryMailKey, key); + } + }) + ); + } + private async sendMailInternal({ startTime, name, @@ -294,9 +326,11 @@ export class MailJob { async onUserDeleted(user: Events['user.deleted']) { await Promise.all([ this.deleteRecipientMailCache(user.email), + this.deleteInvitationMailCacheBySender(user.id), this.queue.removeWhere( 'notification.sendMail', - job => job.to === user.email + job => + job.to === user.email || this.isInvitationMailSentByUser(job, user.id) ), ]); } diff --git a/packages/backend/server/src/models/user.ts b/packages/backend/server/src/models/user.ts index bf21d107ab..7e7b8fe0cd 100644 --- a/packages/backend/server/src/models/user.ts +++ b/packages/backend/server/src/models/user.ts @@ -284,6 +284,10 @@ export class UserModel extends BaseModel { } } + await this.db.workspaceInvitation.deleteMany({ + where: { inviteeUserId: id }, + }); + const user = await this.db.user.delete({ where: { id } }); this.event.emit('user.deleted', { diff --git a/packages/frontend/apps/electron/src/shared/import.ts b/packages/frontend/apps/electron/src/shared/import.ts index e16388eba3..a9bdd32975 100644 --- a/packages/frontend/apps/electron/src/shared/import.ts +++ b/packages/frontend/apps/electron/src/shared/import.ts @@ -2,7 +2,8 @@ export type NativeImportFormat = | 'markdownZip' | 'notionZip' | 'obsidian' - | 'bearZip'; + | 'bearZip' + | 'oneNote'; export type NativeImportBrowserSource = | { kind: 'file'; file: File } diff --git a/packages/frontend/core/src/desktop/dialogs/import/index.tsx b/packages/frontend/core/src/desktop/dialogs/import/index.tsx index 7b05b84208..2bc4f2fd4f 100644 --- a/packages/frontend/core/src/desktop/dialogs/import/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/import/index.tsx @@ -57,7 +57,7 @@ const logger = new DebugLogger('import'); function shouldSnapshotPickedFiles(type: ImportType, acceptType: AcceptType) { if (acceptType === 'Directory' || acceptType === 'Skip') return false; - return !['markdownZip', 'notion', 'bear'].includes(type); + return !['markdownZip', 'notion', 'bear', 'oneNote'].includes(type); } type ImportType = @@ -66,11 +66,19 @@ type ImportType = | 'notion' | 'obsidian' | 'bear' + | 'oneNote' | 'snapshot' | 'html' | 'docx' | 'dotaffinefile'; -type AcceptType = 'Markdown' | 'Zip' | 'Html' | 'Docx' | 'Directory' | 'Skip'; // Skip is used for dotaffinefile +type AcceptType = + | 'Markdown' + | 'Zip' + | 'Html' + | 'Docx' + | 'OneNote' + | 'Directory' + | 'Skip'; // Skip is used for dotaffinefile type Status = 'idle' | 'importing' | 'success' | 'error'; type ImportErrorState = { code: string; @@ -136,6 +144,7 @@ function requireImportService(importService?: ImportService) { type ImportConfig = { fileOptions: { acceptType: AcceptType; multiple: boolean }; + nativeOnly?: boolean; importFunction: (args: ImportFunctionArgs) => Promise; }; @@ -220,6 +229,19 @@ const importOptions = [ testId: 'editor-option-menu-import-bear', type: 'bear' as ImportType, }, + { + key: 'oneNote', + label: 'com.affine.import.onenote', + prefixIcon: ( + + ), + suffixIcon: ( + + ), + suffixTooltip: 'com.affine.import.onenote.tooltip', + testId: 'editor-option-menu-import-onenote', + type: 'oneNote' as ImportType, + }, { key: 'docx', label: 'com.affine.import.docx', @@ -349,6 +371,17 @@ const importConfigs: Record = { ); }, }, + oneNote: { + fileOptions: { acceptType: 'OneNote', multiple: false }, + nativeOnly: true, + importFunction: async ({ files, importService, context }) => { + const file = files.length === 1 ? files[0] : null; + if (!file) { + throw new Error('Expected a single OneNote file'); + } + return requireImportService(importService).importOneNote(file, context); + }, + }, docx: { fileOptions: { acceptType: 'Docx', multiple: false }, importFunction: async ({ docCollection, files }) => { @@ -408,6 +441,7 @@ const ImportOptionItem = ({ suffixTooltip, type, onImport, + disabled, ...props }: { label: string; @@ -416,10 +450,16 @@ const ImportOptionItem = ({ suffixTooltip?: string; type: ImportType; onImport: (type: ImportType) => void; + disabled?: boolean; }) => { const t = useI18n(); return ( -
onImport(type)} {...props}> +
onImport(type)} + aria-disabled={disabled} + {...props} + > {prefixIcon}
{t[label]()}
{suffixIcon && ( @@ -453,18 +493,24 @@ const ImportOptions = ({ suffixTooltip, testId, type, - }) => ( - - ) + }) => { + const disabled = Boolean( + importConfigs[type].nativeOnly && !BUILD_CONFIG.isElectron + ); + return ( + + ); + } )}
@@ -664,6 +710,9 @@ export const ImportDialog = ({ setImportProgress(null); try { const importConfig = importConfigs[type]; + if (importConfig.nativeOnly && !BUILD_CONFIG.isElectron) { + throw new Error(t['com.affine.import.onenote.desktop-only']()); + } const { acceptType, multiple } = importConfig.fileOptions; const files = diff --git a/packages/frontend/core/src/desktop/dialogs/import/styles.css.ts b/packages/frontend/core/src/desktop/dialogs/import/styles.css.ts index 7db866c250..534fc0d641 100644 --- a/packages/frontend/core/src/desktop/dialogs/import/styles.css.ts +++ b/packages/frontend/core/src/desktop/dialogs/import/styles.css.ts @@ -111,6 +111,19 @@ export const importItem = style({ }, }); +export const importItemDisabled = style([ + importItem, + { + opacity: 0.5, + selectors: { + '&:hover': { + background: cssVarV2('button/secondary'), + cursor: 'not-allowed', + }, + }, + }, +]); + export const importItemLabel = style({ display: 'flex', alignItems: 'center', diff --git a/packages/frontend/core/src/modules/import/services/service.ts b/packages/frontend/core/src/modules/import/services/service.ts index 6e0508ab65..d39098a17f 100644 --- a/packages/frontend/core/src/modules/import/services/service.ts +++ b/packages/frontend/core/src/modules/import/services/service.ts @@ -136,6 +136,16 @@ export class ImportService extends Service { return commitService.commitBatch(batch); } + async importOneNote(file: File, context?: ImportRunContext) { + if (!BUILD_CONFIG.isElectron) { + throw new Error('OneNote import is only available in the desktop app.'); + } + const commitService = this.createCommitService({ + organize: true, + }); + return commitNativeImport('oneNote', file, commitService, context); + } + private createCommitService(options: { organize?: boolean; explorerIcon?: boolean; diff --git a/packages/frontend/i18n/src/i18n-completenesses.json b/packages/frontend/i18n/src/i18n-completenesses.json index 1ec160ee33..738d161015 100644 --- a/packages/frontend/i18n/src/i18n-completenesses.json +++ b/packages/frontend/i18n/src/i18n-completenesses.json @@ -1,5 +1,5 @@ { - "ar": 95, + "ar": 94, "ca": 92, "da": 4, "de": 100, @@ -7,9 +7,9 @@ "en": 100, "es-AR": 91, "es-CL": 92, - "es": 91, + "es": 90, "fa": 90, - "fr": 95, + "fr": 94, "hi": 1, "it": 92, "ja": 90, diff --git a/packages/frontend/i18n/src/i18n.gen.ts b/packages/frontend/i18n/src/i18n.gen.ts index bcb9f6e3eb..8c6c552c20 100644 --- a/packages/frontend/i18n/src/i18n.gen.ts +++ b/packages/frontend/i18n/src/i18n.gen.ts @@ -2514,6 +2514,18 @@ export function useAFFiNEI18N(): { * `Import a Notion Markdown export zip with subpages and attachments.` */ ["com.affine.import.notion-markdown.tooltip"](): string; + /** + * `OneNote (Experimental)` + */ + ["com.affine.import.onenote"](): string; + /** + * `Import a OneNote .one, .onetoc2, or .onepkg file. Available in the desktop app.` + */ + ["com.affine.import.onenote.tooltip"](): string; + /** + * `This format importer is available in the AFFiNE desktop app.` + */ + ["com.affine.import.onenote.desktop-only"](): string; /** * `Obsidian Vault (Experimental)` */ diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index c7b89a0e84..0e21497796 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -627,6 +627,9 @@ "com.affine.import.notion.tooltip": "Import your Notion data. Supported import formats: HTML with subpages.", "com.affine.import.notion-markdown": "Notion (Markdown, .zip)", "com.affine.import.notion-markdown.tooltip": "Import a Notion Markdown export zip with subpages and attachments.", + "com.affine.import.onenote": "OneNote (Experimental)", + "com.affine.import.onenote.tooltip": "Import a OneNote .one, .onetoc2, or .onepkg file. Available in the desktop app.", + "com.affine.import.onenote.desktop-only": "This format importer is available in the AFFiNE desktop app.", "com.affine.import.obsidian": "Obsidian Vault (Experimental)", "com.affine.import.obsidian.tooltip": "Import an Obsidian vault. Select a folder to import all notes, images, and assets with wikilinks resolved.", "com.affine.import.snapshot": "Snapshot", diff --git a/packages/frontend/i18n/src/resources/zh-Hans.json b/packages/frontend/i18n/src/resources/zh-Hans.json index 4b567f9a71..7cb85614bf 100644 --- a/packages/frontend/i18n/src/resources/zh-Hans.json +++ b/packages/frontend/i18n/src/resources/zh-Hans.json @@ -625,6 +625,9 @@ "com.affine.import.modal.tip": "如果您希望请求支持其他文件类型,请随时告诉我们", "com.affine.import.notion": "Notion", "com.affine.import.notion.tooltip": "导入您的 Notion 数据。支持导入格式:HTML 带子页面。", + "com.affine.import.onenote": "OneNote(实验性)", + "com.affine.import.onenote.tooltip": "导入 OneNote .one、.onetoc2 或 .onepkg 文件。仅桌面客户端可用。", + "com.affine.import.onenote.desktop-only": "该格式导入器仅在 AFFiNE 桌面客户端可用。", "com.affine.import.obsidian": "Obsidian 仓库(实验性)", "com.affine.import.obsidian.tooltip": "导入 Obsidian 仓库。选择一个文件夹以导入所有笔记、图片和资源,并解析 wiki 链接。", "com.affine.import.snapshot": "快照", diff --git a/packages/frontend/native/Cargo.toml b/packages/frontend/native/Cargo.toml index 8672236105..8110bdab15 100644 --- a/packages/frontend/native/Cargo.toml +++ b/packages/frontend/native/Cargo.toml @@ -9,14 +9,19 @@ crate-type = ["cdylib", "rlib"] [dependencies] affine_common = { workspace = true, features = ["hashcash"] } +affine_doc_loader = { workspace = true } affine_importer = { workspace = true } affine_media_capture = { path = "./media_capture" } affine_nbstore = { workspace = true, features = ["napi"] } affine_sqlite_v1 = { path = "./sqlite_v1" } +hex = { workspace = true } +infer = { workspace = true } napi = { workspace = true } napi-derive = { workspace = true } once_cell = { workspace = true } +onenote_parser = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } sqlx = { workspace = true, default-features = false, features = [ "chrono", "macros", @@ -26,6 +31,7 @@ sqlx = { workspace = true, default-features = false, features = [ ] } thiserror = { workspace = true } tokio = { workspace = true, features = ["full"] } +typed-path = { workspace = true } [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] mermaid-rs-renderer = { workspace = true } diff --git a/packages/frontend/native/src/import/mod.rs b/packages/frontend/native/src/import/mod.rs new file mode 100644 index 0000000000..d18b84530f --- /dev/null +++ b/packages/frontend/native/src/import/mod.rs @@ -0,0 +1,5 @@ +mod onenote; +mod session; + +use onenote::OneNoteImportProvider; +pub use session::*; diff --git a/packages/frontend/native/src/import/onenote/content/mod.rs b/packages/frontend/native/src/import/onenote/content/mod.rs new file mode 100644 index 0000000000..86a2b8c9b2 --- /dev/null +++ b/packages/frontend/native/src/import/onenote/content/mod.rs @@ -0,0 +1,534 @@ +use std::{collections::HashSet, io::Read}; + +mod rich_text; + +use affine_importer::{ImportWarning, ImportedAsset}; +use onenote_parser::{ + contents::{Content, EmbeddedFile, Image, Outline, OutlineElement, OutlineGroup, OutlineItem, Table}, + page::Page, +}; +use rich_text::rich_text_markdown; +use serde_json::Value as JsonValue; +use sha2::{Digest, Sha256}; + +pub(super) fn page_markdown( + page: &Page, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let mut lines = Vec::new(); + for content in page.contents() { + if let Some(outline) = content.outline() { + lines.push(outline_markdown(outline, assets, warnings, source_path)); + } else if let Some(image) = content.image() { + lines.push(image_markdown(image, assets, warnings, source_path)); + } else if let Some(file) = content.embedded_file() { + lines.push(embedded_file_markdown(file, assets, warnings, source_path)); + } else if content.ink().is_some() { + push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content"); + } + } + let markdown = lines + .into_iter() + .filter(|line| !line.trim().is_empty()) + .collect::>() + .join("\n\n"); + if markdown.trim().is_empty() { + " ".to_string() + } else { + markdown + } +} + +pub(super) fn rewrite_non_image_assets_to_attachments(value: &mut JsonValue, assets: &[ImportedAsset]) { + match value { + JsonValue::Object(map) => { + let attachment = map + .get("flavour") + .and_then(JsonValue::as_str) + .filter(|flavour| *flavour == "affine:image") + .and_then(|_| map.get("props")) + .and_then(JsonValue::as_object) + .and_then(|props| props.get("sourceId")) + .and_then(JsonValue::as_str) + .and_then(|source_id| assets.iter().find(|asset| asset.blob_id == source_id)) + .filter(|asset| !asset.mime.starts_with("image/")); + + if let Some(asset) = attachment { + map.insert( + "flavour".to_string(), + JsonValue::String("affine:attachment".to_string()), + ); + if let Some(props) = map.get_mut("props").and_then(JsonValue::as_object_mut) { + props.insert("name".to_string(), JsonValue::String(asset.file_name.clone())); + props.insert("size".to_string(), JsonValue::Number(asset.bytes.len().into())); + props.insert("type".to_string(), JsonValue::String(asset.mime.clone())); + props.insert("embed".to_string(), JsonValue::Bool(false)); + props.insert("style".to_string(), JsonValue::String("horizontalThin".to_string())); + props.insert("footnoteIdentifier".to_string(), JsonValue::Null); + } + } + for value in map.values_mut() { + rewrite_non_image_assets_to_attachments(value, assets); + } + } + JsonValue::Array(values) => { + for value in values { + rewrite_non_image_assets_to_attachments(value, assets); + } + } + _ => {} + } +} + +pub(super) fn remove_blocks_with_source_ids(value: &mut JsonValue, source_ids: &HashSet) { + if source_ids.is_empty() { + return; + } + + match value { + JsonValue::Object(map) => { + for value in map.values_mut() { + remove_blocks_with_source_ids(value, source_ids); + } + } + JsonValue::Array(values) => { + for value in values.iter_mut() { + remove_blocks_with_source_ids(value, source_ids); + } + values.retain(|value| { + block_source_id(value) + .map(|source_id| !source_ids.contains(source_id)) + .unwrap_or(true) + }); + } + _ => {} + } +} + +fn block_source_id(value: &JsonValue) -> Option<&str> { + value + .as_object() + .and_then(|map| map.get("props")) + .and_then(JsonValue::as_object) + .and_then(|props| props.get("sourceId")) + .and_then(JsonValue::as_str) +} + +fn outline_markdown( + outline: &Outline, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + outline + .items() + .iter() + .map(|item| outline_item_markdown(item, 0, assets, warnings, source_path)) + .filter(|item| !item.trim().is_empty()) + .collect::>() + .join("\n\n") +} + +fn outline_item_markdown( + item: &OutlineItem, + depth: usize, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + match item { + OutlineItem::Element(element) => outline_element_markdown(element, depth, assets, warnings, source_path), + OutlineItem::Group(group) => outline_group_markdown(group, depth, assets, warnings, source_path), + } +} + +fn outline_group_markdown( + group: &OutlineGroup, + depth: usize, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let depth = depth + group.child_level() as usize; + group + .outlines() + .iter() + .map(|item| outline_item_markdown(item, depth, assets, warnings, source_path)) + .filter(|item| !item.trim().is_empty()) + .collect::>() + .join("\n") +} + +fn outline_element_markdown( + element: &OutlineElement, + depth: usize, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let mut content = element + .contents() + .iter() + .map(|content| content_markdown(content, assets, warnings, source_path)) + .filter(|content| !content.trim().is_empty()) + .collect::>() + .join("\n\n"); + + if !content.trim().is_empty() && element_is_list(element) { + content = list_item_markdown(element, depth, &content); + } + + let mut lines = Vec::new(); + if !content.trim().is_empty() { + lines.push(content); + } + for child in element.children() { + let child = outline_item_markdown(child, depth + 1, assets, warnings, source_path); + if !child.trim().is_empty() { + lines.push(child); + } + } + lines.join("\n") +} + +fn content_markdown( + content: &Content, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + if let Some(text) = content.rich_text() { + return rich_text_markdown(text); + } + if let Some(table) = content.table() { + return table_markdown(table, assets, warnings, source_path); + } + if let Some(image) = content.image() { + return image_markdown(image, assets, warnings, source_path); + } else if let Some(file) = content.embedded_file() { + return embedded_file_markdown(file, assets, warnings, source_path); + } else if content.ink().is_some() { + push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content"); + } + String::new() +} + +fn table_markdown( + table: &Table, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let rows = table + .contents() + .iter() + .map(|row| { + row + .contents() + .iter() + .map(|cell| { + let markdown = cell + .contents() + .iter() + .map(|element| outline_element_markdown(element, 0, assets, warnings, source_path)) + .collect::>() + .join("\n"); + markdown_table_cell(&markdown) + }) + .collect::>() + }) + .collect::>(); + if rows.is_empty() { + return String::new(); + } + let cols = rows.iter().map(Vec::len).max().unwrap_or(0).max(1); + let mut markdown = String::new(); + let header = rows.first().cloned().unwrap_or_default(); + markdown.push_str(&markdown_row(&header, cols)); + markdown.push('\n'); + markdown.push_str(&markdown_row(&vec!["---".to_string(); cols], cols)); + for row in rows.iter().skip(1) { + markdown.push('\n'); + markdown.push_str(&markdown_row(row, cols)); + } + markdown +} + +fn markdown_row(row: &[String], cols: usize) -> String { + let mut cells = row.to_vec(); + cells.resize(cols, String::new()); + format!("| {} |", cells.join(" | ")) +} + +fn markdown_table_cell(value: &str) -> String { + value + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::>() + .join("
") + .replace('|', "\\|") +} + +fn escape_markdown_label(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' | '[' | ']' | '*' | '_' | '`' => { + output.push('\\'); + output.push(ch); + } + '\n' | '\r' => output.push(' '), + _ => output.push(ch), + } + } + output +} + +fn image_markdown( + image: &Image, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let Some(bytes) = read_optional_blob(image.read(), warnings, source_path, "onenote-image") else { + return image + .alt_text() + .or_else(|| image.text()) + .unwrap_or_default() + .to_string(); + }; + let detected = infer::get(&bytes); + let file_name = image_file_name(image, assets.len() + 1, detected.map(|kind| kind.extension())); + let blob_id = blob_id(&bytes); + let mime = image_mime(image, &file_name, detected.map(|kind| kind.mime_type())); + assets.push(ImportedAsset { + blob_id: blob_id.clone(), + source_path: format!("{source_path}/{file_name}"), + file_name, + mime, + bytes, + }); + let caption = image.alt_text().or_else(|| image.text()).unwrap_or(""); + format!("![{}](blob://{})", escape_markdown_label(caption), blob_id) +} + +fn element_is_list(element: &OutlineElement) -> bool { + !element.list_contents().is_empty() +} + +fn list_item_markdown(element: &OutlineElement, depth: usize, content: &str) -> String { + let marker = list_marker(element); + let indent = " ".repeat(depth); + content + .lines() + .enumerate() + .map(|(index, line)| { + if index == 0 { + format!("{indent}{marker} {line}") + } else if line.trim().is_empty() { + String::new() + } else { + format!("{indent} {line}") + } + }) + .collect::>() + .join("\n") +} + +fn list_marker(element: &OutlineElement) -> &'static str { + let ordered = element.list_contents().iter().any(|list| { + list.list_restart().is_some() + || list + .list_format() + .iter() + .any(|ch| ch.is_ascii_digit() || matches!(ch, '%' | '#')) + }); + if ordered { "1." } else { "-" } +} + +fn image_file_name(image: &Image, index: usize, detected_ext: Option<&'static str>) -> String { + let raw_name = image + .image_filename() + .filter(|name| !name.trim().is_empty()) + .map(|name| sanitize_file_name(name, "image")) + .unwrap_or_else(|| { + let ext = image.extension().or(detected_ext).unwrap_or("png"); + format!("image-{index}.{ext}") + }); + + if mime_from_path(&raw_name).starts_with("image/") { + return raw_name; + } + + image + .extension() + .or(detected_ext) + .map(|ext| format!("{raw_name}.{ext}")) + .unwrap_or(raw_name) +} + +fn embedded_file_markdown( + file: &EmbeddedFile, + assets: &mut Vec, + warnings: &mut Vec, + source_path: &str, +) -> String { + let mut reader = file.read(); + let mut bytes = Vec::new(); + if let Err(error) = reader.read_to_end(&mut bytes) { + push_warning( + warnings, + source_path, + "onenote-attachment", + &format!("Skipped OneNote embedded file content: {error}"), + ); + return file.filename().to_string(); + } + let file_name = sanitize_file_name(file.filename(), "attachment"); + let blob_id = blob_id(&bytes); + assets.push(ImportedAsset { + blob_id: blob_id.clone(), + source_path: format!("{source_path}/{file_name}"), + file_name: file_name.clone(), + mime: mime_from_path(&file_name).to_string(), + bytes, + }); + format!("![{}](blob://{})", escape_markdown_label(&file_name), blob_id) +} + +fn read_optional_blob( + reader: Option>, + warnings: &mut Vec, + source_path: &str, + code: &str, +) -> Option> { + let Some(mut reader) = reader else { + push_warning( + warnings, + source_path, + code, + "Skipped OneNote blob content: missing data", + ); + return None; + }; + let mut bytes = Vec::new(); + if let Err(error) = reader.read_to_end(&mut bytes) { + push_warning( + warnings, + source_path, + code, + &format!("Skipped OneNote blob content: {error}"), + ); + return None; + } + Some(bytes) +} + +fn image_mime(image: &Image, file_name: &str, detected_mime: Option<&'static str>) -> String { + image + .extension() + .map(mime_from_extension) + .or(detected_mime) + .unwrap_or_else(|| mime_from_path(file_name)) + .to_string() +} + +fn blob_id(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + hex::encode(hasher.finalize()) +} + +fn sanitize_file_name(value: &str, fallback: &str) -> String { + let name = value + .split(['/', '\\', '\0']) + .filter(|part| !part.is_empty() && *part != "." && *part != "..") + .collect::>() + .join(" ") + .trim() + .to_string(); + if name.is_empty() { fallback.to_string() } else { name } +} + +fn mime_from_path(path: &str) -> &'static str { + match path.rsplit('.').next().unwrap_or("").to_lowercase().as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "webp" => "image/webp", + "bmp" => "image/bmp", + "svg" => "image/svg+xml", + "pdf" => "application/pdf", + "txt" => "text/plain", + "md" | "markdown" => "text/markdown", + "csv" => "text/csv", + "zip" => "application/zip", + "doc" => "application/msword", + "docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls" => "application/vnd.ms-excel", + "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "ppt" => "application/vnd.ms-powerpoint", + "pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation", + _ => "application/octet-stream", + } +} + +fn mime_from_extension(extension: &str) -> &'static str { + mime_from_path(extension) +} + +fn push_warning(warnings: &mut Vec, source_path: &str, code: &str, message: &str) { + warnings.push(ImportWarning { + code: code.to_string(), + source_path: Some(source_path.to_string()), + message: message.to_string(), + }); +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn markdown_table_cell_collapses_multiline_content() { + assert_eq!( + markdown_table_cell("first line\nsecond | line\n\n third line "), + "first line
second \\| line
third line" + ); + } + + #[test] + fn remove_blocks_with_source_ids_removes_matching_nested_blocks() { + let mut snapshot = json!({ + "children": [ + { + "flavour": "affine:paragraph", + "props": {}, + "children": [ + { + "flavour": "affine:image", + "props": { "sourceId": "skipped" }, + "children": [] + }, + { + "flavour": "affine:image", + "props": { "sourceId": "kept" }, + "children": [] + } + ] + } + ] + }); + let source_ids = HashSet::from(["skipped".to_string()]); + + remove_blocks_with_source_ids(&mut snapshot, &source_ids); + + assert_eq!(snapshot["children"][0]["children"].as_array().unwrap().len(), 1); + assert_eq!(snapshot["children"][0]["children"][0]["props"]["sourceId"], "kept"); + } +} diff --git a/packages/frontend/native/src/import/onenote/content/rich_text.rs b/packages/frontend/native/src/import/onenote/content/rich_text.rs new file mode 100644 index 0000000000..ee63589a33 --- /dev/null +++ b/packages/frontend/native/src/import/onenote/content/rich_text.rs @@ -0,0 +1,171 @@ +use onenote_parser::{ + contents::{ParagraphStyling, RichText}, + property::common::ColorRef, +}; + +pub(super) fn rich_text_markdown(text: &RichText) -> String { + let mut markdown = styled_text_markdown(text); + let trimmed = markdown.trim(); + if trimmed.is_empty() { + return String::new(); + } + + if let Some(prefix) = heading_prefix(text, trimmed) { + markdown = format!("{prefix} {}", trimmed.trim_start_matches('#').trim_start()); + } + markdown +} + +fn styled_text_markdown(text: &RichText) -> String { + let chars = text.text().chars().collect::>(); + let styles = text.text_run_formatting(); + if chars.is_empty() { + return String::new(); + } + if styles.is_empty() { + return chars.into_iter().collect(); + } + + let mut output = String::new(); + let mut start = 0usize; + let mut pending_link: Option = None; + for (index, style) in styles.iter().enumerate() { + let end = text + .text_run_indices() + .get(index) + .map(|end| (*end as usize).min(chars.len())) + .unwrap_or(chars.len()); + if end < start { + continue; + } + let segment = chars[start..end].iter().collect::(); + start = end; + if segment.is_empty() { + continue; + } + if style.hidden() { + if let Some(url) = extract_hyperlink_url(&segment) { + pending_link = Some(url); + } + continue; + } + output.push_str(&format_segment(&segment, style, &mut pending_link)); + } + if start < chars.len() { + output.push_str(&chars[start..].iter().collect::()); + } + output +} + +fn heading_prefix(text: &RichText, value: &str) -> Option<&'static str> { + if value.chars().count() > 120 || value.lines().count() > 1 { + return None; + } + let font_size = text + .text_run_formatting() + .iter() + .filter(|style| !style.hidden()) + .filter_map(ParagraphStyling::font_size) + .max() + .or_else(|| text.paragraph_style().font_size()); + + match font_size { + Some(size) if size >= 28 => Some("##"), + Some(size) if size >= 24 => Some("###"), + _ => None, + } +} + +fn format_segment(segment: &str, style: &ParagraphStyling, pending_link: &mut Option) -> String { + let mut value = escape_html(segment); + if segment.trim().is_empty() { + return value; + } + + if style.bold() && should_keep_inline_style(segment) { + value = wrap_inline(&value, "", ""); + } + if style.italic() && should_keep_inline_style(segment) { + value = wrap_inline(&value, "", ""); + } + if style.strikethrough() { + value = wrap_inline(&value, "", ""); + } + if style.underline() { + value = wrap_inline(&value, "", ""); + } + if let Some(color) = style.font_color().and_then(hex_color) { + value = wrap_inline(&value, &format!(""), ""); + } + if let Some(color) = style.highlight().and_then(hex_color) { + value = wrap_inline(&value, &format!(""), ""); + } + + if style.hyperlink_protected() + && let Some(url) = pending_link.take() + { + return wrap_inline(&value, &format!("", escape_html_attr(&url)), ""); + } + value +} + +fn should_keep_inline_style(value: &str) -> bool { + value.chars().any(|ch| ch.is_alphanumeric()) +} + +fn wrap_inline(value: &str, open: &str, close: &str) -> String { + let leading_len = value.len() - value.trim_start().len(); + let trailing_len = value.len() - value.trim_end().len(); + let (leading, rest) = value.split_at(leading_len); + let (middle, trailing) = rest.split_at(rest.len().saturating_sub(trailing_len)); + if middle.is_empty() { + value.to_string() + } else { + format!("{leading}{open}{middle}{close}{trailing}") + } +} + +fn hex_color(color: ColorRef) -> Option { + match color { + ColorRef::Auto => None, + ColorRef::Manual { r, g, b } => Some(format!("#{r:02x}{g:02x}{b:02x}")), + } +} + +fn escape_html(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + _ => output.push(ch), + } + } + output +} + +fn escape_html_attr(value: &str) -> String { + let mut output = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '&' => output.push_str("&"), + '<' => output.push_str("<"), + '>' => output.push_str(">"), + '"' => output.push_str("""), + _ => output.push(ch), + } + } + output +} + +fn extract_hyperlink_url(value: &str) -> Option { + let marker = "HYPERLINK"; + let start = value.find(marker)? + marker.len(); + let rest = value[start..].trim(); + if let Some(quoted) = rest.strip_prefix('"') { + let end = quoted.find('"')?; + return Some(quoted[..end].to_string()); + } + rest.split_whitespace().next().map(ToString::to_string) +} diff --git a/packages/frontend/native/src/import/onenote/mod.rs b/packages/frontend/native/src/import/onenote/mod.rs new file mode 100644 index 0000000000..53f65c69a2 --- /dev/null +++ b/packages/frontend/native/src/import/onenote/mod.rs @@ -0,0 +1,444 @@ +use std::{ + collections::{HashSet, hash_map::DefaultHasher}, + fs, + hash::{Hash, Hasher}, + path::{Path, PathBuf}, +}; + +mod content; + +use affine_importer::{ + FolderHierarchyDelta, ImportBatch, ImportBatchLimits, ImportCursor, ImportError, ImportOptions, ImportProgress, + ImportProvider, ImportResult, ImportSource, ImportWarning, ImportedAsset, ImportedDocMeta, ImportedDocSnapshot, +}; +use content::{page_markdown, remove_blocks_with_source_ids, rewrite_non_image_assets_to_attachments}; +use onenote_parser::{ + Parser, + notebook::Notebook, + page::Page, + section::{Section, SectionEntry, SectionGroup}, +}; +use typed_path::TypedPath; + +pub struct OneNoteImportProvider; + +impl OneNoteImportProvider { + pub fn new() -> Self { + Self + } +} + +impl ImportProvider for OneNoteImportProvider { + fn format(&self) -> &'static str { + "oneNote" + } + + fn create_cursor( + &self, + source: ImportSource, + _options: ImportOptions, + limits: ImportBatchLimits, + ) -> ImportResult> { + let mut planner = OneNotePlanner { + docs: Vec::new(), + folders: Vec::new(), + warnings: Vec::new(), + }; + planner.read_source(source)?; + Ok(Box::new(OneNoteImportCursor { + docs: planner.docs, + folders: planner.folders, + warnings: planner.warnings, + next_asset_doc: 0, + next_asset_index: 0, + next_doc: 0, + emitted_final: false, + skipped_blob_ids: HashSet::new(), + limits, + })) + } +} + +struct OneNoteDoc { + id: String, + title: String, + markdown: String, + assets: Vec, +} + +struct OneNotePlanner { + docs: Vec, + folders: Vec, + warnings: Vec, +} + +impl OneNotePlanner { + fn read_source(&mut self, source: ImportSource) -> ImportResult<()> { + match source { + ImportSource::FilePath(path) => self.read_file(&path), + ImportSource::DirectoryPath(path) => self.read_directory(&path), + } + } + + fn read_file(&mut self, path: &Path) -> ImportResult<()> { + let parser = Parser::new(); + match extension(path).as_deref() { + Some("one") => { + let source_path = path.to_string_lossy(); + let section = parser + .parse_section(TypedPath::derive(source_path.as_ref())) + .map_err(import_error)?; + self.add_section(§ion, None); + Ok(()) + } + Some("onetoc2") => { + let source_path = path.to_string_lossy(); + let notebook = parser + .parse_notebook(TypedPath::derive(source_path.as_ref())) + .map_err(import_error)?; + self.add_notebook(¬ebook, notebook_name(path)); + Ok(()) + } + Some("onepkg") => { + let source_path = path.to_string_lossy(); + let notebook = parser + .parse_package(TypedPath::derive(source_path.as_ref())) + .map_err(import_error)?; + self.add_notebook(¬ebook, notebook_name(path)); + Ok(()) + } + _ => Err(ImportError::InvalidSource(format!( + "unsupported OneNote source: {}", + path.to_string_lossy() + ))), + } + } + + fn read_directory(&mut self, path: &Path) -> ImportResult<()> { + let toc = find_first_onetoc2(path)?.ok_or_else(|| { + ImportError::InvalidSource(format!( + "directory contains no .onetoc2 file: {}", + path.to_string_lossy() + )) + })?; + let parser = Parser::new(); + let toc_path = toc.to_string_lossy(); + let notebook = parser + .parse_notebook(TypedPath::derive(toc_path.as_ref())) + .map_err(import_error)?; + self.add_notebook(¬ebook, notebook_name(path)); + Ok(()) + } + + fn add_notebook(&mut self, notebook: &Notebook, name: String) { + let root = sanitize_path_part(&name); + self.folders.push(FolderHierarchyDelta { + path: root.clone(), + name, + parent_path: None, + page_id: None, + icon: None, + }); + for entry in notebook.entries() { + self.add_section_entry(entry, Some(root.clone())); + } + } + + fn add_section_entry(&mut self, entry: &SectionEntry, parent_path: Option) { + match entry { + SectionEntry::Section(section) => self.add_section(section, parent_path), + SectionEntry::SectionGroup(group) => self.add_section_group(group, parent_path), + } + } + + fn add_section_group(&mut self, group: &SectionGroup, parent_path: Option) { + let path = join_import_path(parent_path.as_deref(), group.display_name()); + self.folders.push(FolderHierarchyDelta { + path: path.clone(), + name: group.display_name().to_string(), + parent_path, + page_id: None, + icon: None, + }); + for entry in group.entries() { + self.add_section_entry(entry, Some(path.clone())); + } + } + + fn add_section(&mut self, section: &Section, parent_path: Option) { + let path = join_import_path(parent_path.as_deref(), section.display_name()); + self.folders.push(FolderHierarchyDelta { + path: path.clone(), + name: section.display_name().to_string(), + parent_path, + page_id: None, + icon: None, + }); + let mut page_index = 0; + for series in section.page_series() { + for page in series.pages() { + self.add_page(page, &path, page_index); + page_index += 1; + } + } + } + + fn add_page(&mut self, page: &Page, parent_path: &str, page_index: usize) { + let title = page + .title_text() + .map(str::trim) + .filter(|title| !title.is_empty()) + .unwrap_or("Untitled") + .to_string(); + let source_path = join_import_path(Some(parent_path), &title); + let id = doc_id(page.link_target_id(), &title, parent_path, page_index); + let mut assets = Vec::new(); + let markdown = page_markdown(page, &mut assets, &mut self.warnings, &source_path); + self.folders.push(FolderHierarchyDelta { + path: source_path.clone(), + name: title.clone(), + parent_path: Some(parent_path.to_string()), + page_id: Some(id.clone()), + icon: None, + }); + self.docs.push(OneNoteDoc { + id, + title, + markdown, + assets, + }); + } +} + +struct OneNoteImportCursor { + docs: Vec, + folders: Vec, + warnings: Vec, + next_asset_doc: usize, + next_asset_index: usize, + next_doc: usize, + emitted_final: bool, + skipped_blob_ids: HashSet, + limits: ImportBatchLimits, +} + +impl ImportCursor for OneNoteImportCursor { + fn total(&self) -> usize { + self.docs.len() + } + + fn next_batch(&mut self) -> ImportResult> { + if self.next_doc >= self.docs.len() && self.emitted_final { + return Ok(None); + } + + if let Some(batch) = self.next_asset_batch() { + return Ok(Some(batch)); + } + + let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.docs.len()); + let mut batch = empty_batch(self.docs.len()); + for doc in &self.docs[self.next_doc..end] { + let mut snapshot = affine_doc_loader::build_doc_snapshot(&doc.title, &doc.markdown, &doc.id)?; + rewrite_non_image_assets_to_attachments(&mut snapshot, &doc.assets); + remove_blocks_with_source_ids(&mut snapshot, &self.skipped_blob_ids); + batch.docs.push(ImportedDocSnapshot { + id: doc.id.clone(), + snapshot, + meta: Some(ImportedDocMeta { + title: Some(doc.title.clone()), + create_date: None, + updated_date: None, + tags: None, + favorite: None, + trash: None, + }), + }); + } + self.next_doc = end; + if self.next_doc >= self.docs.len() { + batch.folders = std::mem::take(&mut self.folders); + batch.warnings = std::mem::take(&mut self.warnings); + self.emitted_final = true; + } + batch.progress = ImportProgress { + completed: self.next_doc, + total: self.docs.len(), + }; + batch.done = self.emitted_final; + Ok(Some(batch)) + } +} + +impl OneNoteImportCursor { + fn next_asset_batch(&mut self) -> Option { + let max_blobs = self.limits.max_blobs.max(1); + let max_blob_bytes = self.limits.max_blob_bytes; + let mut batch = empty_batch(self.docs.len()); + let mut bytes_in_batch = 0u64; + + while self.next_asset_doc < self.docs.len() && batch.blobs.len() < max_blobs { + let doc = &self.docs[self.next_asset_doc]; + if self.next_asset_index >= doc.assets.len() { + self.next_asset_doc += 1; + self.next_asset_index = 0; + continue; + } + + let asset = &doc.assets[self.next_asset_index]; + self.next_asset_index += 1; + let asset_size = asset.bytes.len() as u64; + if asset_size > max_blob_bytes { + self.skipped_blob_ids.insert(asset.blob_id.clone()); + batch.warnings.push(ImportWarning { + code: "blob_too_large".to_string(), + source_path: Some(asset.source_path.clone()), + message: format!("Skipped {}: blob is larger than batch limit", asset.source_path), + }); + continue; + } + if !batch.blobs.is_empty() && bytes_in_batch + asset_size > max_blob_bytes { + self.next_asset_index -= 1; + break; + } + bytes_in_batch += asset_size; + batch.blobs.push(asset.clone()); + } + + if batch.blobs.is_empty() && batch.warnings.is_empty() { + return None; + } + batch.progress = ImportProgress { + completed: self.next_doc, + total: self.docs.len(), + }; + Some(batch) + } +} + +fn empty_batch(total: usize) -> ImportBatch { + ImportBatch { + docs: Vec::new(), + blobs: Vec::new(), + folders: Vec::new(), + tags: Vec::new(), + icons: Vec::new(), + warnings: Vec::new(), + progress: ImportProgress { completed: 0, total }, + entry_id: None, + is_workspace_file: false, + done: false, + } +} + +fn find_first_onetoc2(root: &Path) -> ImportResult> { + for entry in fs::read_dir(root)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + if let Some(found) = find_first_onetoc2(&path)? { + return Ok(Some(found)); + } + } else if extension(&path).as_deref() == Some("onetoc2") { + return Ok(Some(path)); + } + } + Ok(None) +} + +fn import_error(error: onenote_parser::errors::Error) -> ImportError { + ImportError::InvalidSource(error.to_string()) +} + +fn notebook_name(path: &Path) -> String { + path + .file_stem() + .or_else(|| path.file_name()) + .map(|name| name.to_string_lossy().to_string()) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| "OneNote".to_string()) +} + +fn extension(path: &Path) -> Option { + path.extension().map(|ext| ext.to_string_lossy().to_lowercase()) +} + +fn join_import_path(parent: Option<&str>, name: &str) -> String { + let name = sanitize_path_part(name); + parent + .filter(|parent| !parent.is_empty()) + .map(|parent| format!("{parent}/{name}")) + .unwrap_or(name) +} + +fn sanitize_path_part(value: &str) -> String { + let name = value + .split(['/', '\\']) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" ") + .trim() + .to_string(); + if name.is_empty() { "Untitled".to_string() } else { name } +} + +fn doc_id(raw_id: &str, title: &str, parent_path: &str, page_index: usize) -> String { + let mut id = raw_id + .chars() + .filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_') + .collect::(); + if id.is_empty() { + let slug = title + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .take(32) + .collect::(); + let mut hasher = DefaultHasher::new(); + title.hash(&mut hasher); + parent_path.hash(&mut hasher); + page_index.hash(&mut hasher); + let hash = format!("{:x}", hasher.finish()); + id = if slug.is_empty() { + hash + } else { + format!("{slug}-{hash}") + }; + } + format!("onenote-{id}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_path_part_removes_path_separators() { + assert_eq!(sanitize_path_part("Group/Section\\Page"), "Group Section Page"); + assert_eq!(sanitize_path_part("///"), "Untitled"); + } + + #[test] + fn join_import_path_sanitizes_path_parts() { + assert_eq!( + join_import_path(Some("Notebook/Section"), "Page/Subpage\\Draft"), + "Notebook/Section/Page Subpage Draft" + ); + } + + #[test] + fn doc_id_keeps_ascii_ids_and_hashes_non_ascii_titles() { + assert_eq!(doc_id("{abc-123}", "Ignored", "Notebook/Section", 0), "onenote-abc-123"); + + let id = doc_id("", "中文标题", "Notebook/Section", 0); + assert!(id.starts_with("onenote-")); + assert!(id.len() > "onenote-".len()); + } + + #[test] + fn doc_id_fallback_distinguishes_duplicate_titles() { + let first = doc_id("", "Untitled", "Notebook/Section", 0); + let second = doc_id("", "Untitled", "Notebook/Section", 1); + + assert_ne!(first, second); + } +} diff --git a/packages/frontend/native/src/import_session.rs b/packages/frontend/native/src/import/session.rs similarity index 84% rename from packages/frontend/native/src/import_session.rs rename to packages/frontend/native/src/import/session.rs index 47b63aa3a5..8d6e817678 100644 --- a/packages/frontend/native/src/import_session.rs +++ b/packages/frontend/native/src/import/session.rs @@ -2,13 +2,13 @@ use std::{ collections::HashMap, path::PathBuf, sync::{ - Mutex, + Arc, Mutex, atomic::{AtomicU64, Ordering}, }, }; use affine_importer::{ - ImportBatchLimits, ImportFormat, ImportOptions, ImportSession, ImportSessionOptions, ImportSessionSource, + ImportBatchLimits, ImportOptions, ImportRegistry, ImportSession, ImportSessionOptions, ImportSource, }; use napi::{Status, bindgen_prelude::*}; use napi_derive::napi; @@ -16,6 +16,11 @@ use once_cell::sync::Lazy; static IMPORT_SESSIONS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1); +static IMPORT_REGISTRY: Lazy> = Lazy::new(|| { + let mut registry = ImportRegistry::with_builtin(); + registry.register(super::OneNoteImportProvider::new()); + Arc::new(registry) +}); #[napi(object)] pub struct CreateImportSessionOptions { @@ -37,25 +42,10 @@ pub struct CreateImportBatchLimits { pub max_blob_bytes: Option, } -fn parse_format(format: &str) -> Result { - match format { - "markdownZip" => Ok(ImportFormat::MarkdownZip), - "notionZip" => Ok(ImportFormat::NotionZip), - "notionMarkdownZip" => Ok(ImportFormat::NotionMarkdownZip), - "notionHtmlZip" => Ok(ImportFormat::NotionHtmlZip), - "obsidian" => Ok(ImportFormat::Obsidian), - "bearZip" => Ok(ImportFormat::BearZip), - _ => Err(Error::new( - Status::InvalidArg, - format!("unsupported import format: {format}"), - )), - } -} - fn map_import_error(error: affine_importer::ImportError) -> Error { let status = match error { affine_importer::ImportError::Cancelled => Status::Cancelled, - affine_importer::ImportError::UnsupportedFormat | affine_importer::ImportError::InvalidSource(_) => { + affine_importer::ImportError::UnsupportedFormat(_) | affine_importer::ImportError::InvalidSource(_) => { Status::InvalidArg } affine_importer::ImportError::Zip(_) @@ -67,10 +57,9 @@ fn map_import_error(error: affine_importer::ImportError) -> Error { #[napi] pub fn create_import_session(options: CreateImportSessionOptions) -> Result { - let format = parse_format(&options.format)?; let source = match options.source.kind.as_str() { - "filePath" => ImportSessionSource::FilePath(PathBuf::from(options.source.path)), - "directoryPath" => ImportSessionSource::DirectoryPath(PathBuf::from(options.source.path)), + "filePath" => ImportSource::FilePath(PathBuf::from(options.source.path)), + "directoryPath" => ImportSource::DirectoryPath(PathBuf::from(options.source.path)), kind => { return Err(Error::new( Status::InvalidArg, @@ -93,12 +82,15 @@ pub fn create_import_session(options: CreateImportSessionOptions) -> Result