feat(core): onenote importer (#15198)

#### PR Dependency Tree


* **PR #15198** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added OneNote import support for `.one`, `.onetoc2`, and `.onepkg`,
including OneNote-to-markdown content conversion.
* OneNote now appears in the import flow with a dedicated label and
format tooltip, and file pickers recognize the new OneNote type.

* **Bug Fixes**
* Import options that are desktop-only are now disabled when not running
in the desktop app, with clear messaging.
* Improved imported asset handling by converting non-image embedded
assets into attachments for more consistent results.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-06 06:20:52 +08:00
committed by GitHub
parent 8d72e4dc29
commit 9581432d21
22 changed files with 1621 additions and 112 deletions
@@ -0,0 +1,5 @@
mod onenote;
mod session;
use onenote::OneNoteImportProvider;
pub use session::*;
@@ -0,0 +1,534 @@
use std::{collections::HashSet, io::Read};
mod rich_text;
use affine_importer::{ImportWarning, ImportedAsset};
use onenote_parser::{
contents::{Content, EmbeddedFile, Image, Outline, OutlineElement, OutlineGroup, OutlineItem, Table},
page::Page,
};
use rich_text::rich_text_markdown;
use serde_json::Value as JsonValue;
use sha2::{Digest, Sha256};
pub(super) fn page_markdown(
page: &Page,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
let mut lines = Vec::new();
for content in page.contents() {
if let Some(outline) = content.outline() {
lines.push(outline_markdown(outline, assets, warnings, source_path));
} else if let Some(image) = content.image() {
lines.push(image_markdown(image, assets, warnings, source_path));
} else if let Some(file) = content.embedded_file() {
lines.push(embedded_file_markdown(file, assets, warnings, source_path));
} else if content.ink().is_some() {
push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content");
}
}
let markdown = lines
.into_iter()
.filter(|line| !line.trim().is_empty())
.collect::<Vec<_>>()
.join("\n\n");
if markdown.trim().is_empty() {
" ".to_string()
} else {
markdown
}
}
pub(super) fn rewrite_non_image_assets_to_attachments(value: &mut JsonValue, assets: &[ImportedAsset]) {
match value {
JsonValue::Object(map) => {
let attachment = map
.get("flavour")
.and_then(JsonValue::as_str)
.filter(|flavour| *flavour == "affine:image")
.and_then(|_| map.get("props"))
.and_then(JsonValue::as_object)
.and_then(|props| props.get("sourceId"))
.and_then(JsonValue::as_str)
.and_then(|source_id| assets.iter().find(|asset| asset.blob_id == source_id))
.filter(|asset| !asset.mime.starts_with("image/"));
if let Some(asset) = attachment {
map.insert(
"flavour".to_string(),
JsonValue::String("affine:attachment".to_string()),
);
if let Some(props) = map.get_mut("props").and_then(JsonValue::as_object_mut) {
props.insert("name".to_string(), JsonValue::String(asset.file_name.clone()));
props.insert("size".to_string(), JsonValue::Number(asset.bytes.len().into()));
props.insert("type".to_string(), JsonValue::String(asset.mime.clone()));
props.insert("embed".to_string(), JsonValue::Bool(false));
props.insert("style".to_string(), JsonValue::String("horizontalThin".to_string()));
props.insert("footnoteIdentifier".to_string(), JsonValue::Null);
}
}
for value in map.values_mut() {
rewrite_non_image_assets_to_attachments(value, assets);
}
}
JsonValue::Array(values) => {
for value in values {
rewrite_non_image_assets_to_attachments(value, assets);
}
}
_ => {}
}
}
pub(super) fn remove_blocks_with_source_ids(value: &mut JsonValue, source_ids: &HashSet<String>) {
if source_ids.is_empty() {
return;
}
match value {
JsonValue::Object(map) => {
for value in map.values_mut() {
remove_blocks_with_source_ids(value, source_ids);
}
}
JsonValue::Array(values) => {
for value in values.iter_mut() {
remove_blocks_with_source_ids(value, source_ids);
}
values.retain(|value| {
block_source_id(value)
.map(|source_id| !source_ids.contains(source_id))
.unwrap_or(true)
});
}
_ => {}
}
}
fn block_source_id(value: &JsonValue) -> Option<&str> {
value
.as_object()
.and_then(|map| map.get("props"))
.and_then(JsonValue::as_object)
.and_then(|props| props.get("sourceId"))
.and_then(JsonValue::as_str)
}
fn outline_markdown(
outline: &Outline,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
outline
.items()
.iter()
.map(|item| outline_item_markdown(item, 0, assets, warnings, source_path))
.filter(|item| !item.trim().is_empty())
.collect::<Vec<_>>()
.join("\n\n")
}
fn outline_item_markdown(
item: &OutlineItem,
depth: usize,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
match item {
OutlineItem::Element(element) => outline_element_markdown(element, depth, assets, warnings, source_path),
OutlineItem::Group(group) => outline_group_markdown(group, depth, assets, warnings, source_path),
}
}
fn outline_group_markdown(
group: &OutlineGroup,
depth: usize,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
let depth = depth + group.child_level() as usize;
group
.outlines()
.iter()
.map(|item| outline_item_markdown(item, depth, assets, warnings, source_path))
.filter(|item| !item.trim().is_empty())
.collect::<Vec<_>>()
.join("\n")
}
fn outline_element_markdown(
element: &OutlineElement,
depth: usize,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
let mut content = element
.contents()
.iter()
.map(|content| content_markdown(content, assets, warnings, source_path))
.filter(|content| !content.trim().is_empty())
.collect::<Vec<_>>()
.join("\n\n");
if !content.trim().is_empty() && element_is_list(element) {
content = list_item_markdown(element, depth, &content);
}
let mut lines = Vec::new();
if !content.trim().is_empty() {
lines.push(content);
}
for child in element.children() {
let child = outline_item_markdown(child, depth + 1, assets, warnings, source_path);
if !child.trim().is_empty() {
lines.push(child);
}
}
lines.join("\n")
}
fn content_markdown(
content: &Content,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
if let Some(text) = content.rich_text() {
return rich_text_markdown(text);
}
if let Some(table) = content.table() {
return table_markdown(table, assets, warnings, source_path);
}
if let Some(image) = content.image() {
return image_markdown(image, assets, warnings, source_path);
} else if let Some(file) = content.embedded_file() {
return embedded_file_markdown(file, assets, warnings, source_path);
} else if content.ink().is_some() {
push_warning(warnings, source_path, "onenote-ink", "Skipped OneNote ink content");
}
String::new()
}
fn table_markdown(
table: &Table,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
let rows = table
.contents()
.iter()
.map(|row| {
row
.contents()
.iter()
.map(|cell| {
let markdown = cell
.contents()
.iter()
.map(|element| outline_element_markdown(element, 0, assets, warnings, source_path))
.collect::<Vec<_>>()
.join("\n");
markdown_table_cell(&markdown)
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
if rows.is_empty() {
return String::new();
}
let cols = rows.iter().map(Vec::len).max().unwrap_or(0).max(1);
let mut markdown = String::new();
let header = rows.first().cloned().unwrap_or_default();
markdown.push_str(&markdown_row(&header, cols));
markdown.push('\n');
markdown.push_str(&markdown_row(&vec!["---".to_string(); cols], cols));
for row in rows.iter().skip(1) {
markdown.push('\n');
markdown.push_str(&markdown_row(row, cols));
}
markdown
}
fn markdown_row(row: &[String], cols: usize) -> String {
let mut cells = row.to_vec();
cells.resize(cols, String::new());
format!("| {} |", cells.join(" | "))
}
fn markdown_table_cell(value: &str) -> String {
value
.lines()
.map(str::trim)
.filter(|line| !line.is_empty())
.collect::<Vec<_>>()
.join("<br />")
.replace('|', "\\|")
}
fn escape_markdown_label(value: &str) -> String {
let mut output = String::with_capacity(value.len());
for ch in value.chars() {
match ch {
'\\' | '[' | ']' | '*' | '_' | '`' => {
output.push('\\');
output.push(ch);
}
'\n' | '\r' => output.push(' '),
_ => output.push(ch),
}
}
output
}
fn image_markdown(
image: &Image,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
let Some(bytes) = read_optional_blob(image.read(), warnings, source_path, "onenote-image") else {
return image
.alt_text()
.or_else(|| image.text())
.unwrap_or_default()
.to_string();
};
let detected = infer::get(&bytes);
let file_name = image_file_name(image, assets.len() + 1, detected.map(|kind| kind.extension()));
let blob_id = blob_id(&bytes);
let mime = image_mime(image, &file_name, detected.map(|kind| kind.mime_type()));
assets.push(ImportedAsset {
blob_id: blob_id.clone(),
source_path: format!("{source_path}/{file_name}"),
file_name,
mime,
bytes,
});
let caption = image.alt_text().or_else(|| image.text()).unwrap_or("");
format!("![{}](blob://{})", escape_markdown_label(caption), blob_id)
}
fn element_is_list(element: &OutlineElement) -> bool {
!element.list_contents().is_empty()
}
fn list_item_markdown(element: &OutlineElement, depth: usize, content: &str) -> String {
let marker = list_marker(element);
let indent = " ".repeat(depth);
content
.lines()
.enumerate()
.map(|(index, line)| {
if index == 0 {
format!("{indent}{marker} {line}")
} else if line.trim().is_empty() {
String::new()
} else {
format!("{indent} {line}")
}
})
.collect::<Vec<_>>()
.join("\n")
}
fn list_marker(element: &OutlineElement) -> &'static str {
let ordered = element.list_contents().iter().any(|list| {
list.list_restart().is_some()
|| list
.list_format()
.iter()
.any(|ch| ch.is_ascii_digit() || matches!(ch, '%' | '#'))
});
if ordered { "1." } else { "-" }
}
fn image_file_name(image: &Image, index: usize, detected_ext: Option<&'static str>) -> String {
let raw_name = image
.image_filename()
.filter(|name| !name.trim().is_empty())
.map(|name| sanitize_file_name(name, "image"))
.unwrap_or_else(|| {
let ext = image.extension().or(detected_ext).unwrap_or("png");
format!("image-{index}.{ext}")
});
if mime_from_path(&raw_name).starts_with("image/") {
return raw_name;
}
image
.extension()
.or(detected_ext)
.map(|ext| format!("{raw_name}.{ext}"))
.unwrap_or(raw_name)
}
fn embedded_file_markdown(
file: &EmbeddedFile,
assets: &mut Vec<ImportedAsset>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
) -> String {
let mut reader = file.read();
let mut bytes = Vec::new();
if let Err(error) = reader.read_to_end(&mut bytes) {
push_warning(
warnings,
source_path,
"onenote-attachment",
&format!("Skipped OneNote embedded file content: {error}"),
);
return file.filename().to_string();
}
let file_name = sanitize_file_name(file.filename(), "attachment");
let blob_id = blob_id(&bytes);
assets.push(ImportedAsset {
blob_id: blob_id.clone(),
source_path: format!("{source_path}/{file_name}"),
file_name: file_name.clone(),
mime: mime_from_path(&file_name).to_string(),
bytes,
});
format!("![{}](blob://{})", escape_markdown_label(&file_name), blob_id)
}
fn read_optional_blob(
reader: Option<Box<dyn Read>>,
warnings: &mut Vec<ImportWarning>,
source_path: &str,
code: &str,
) -> Option<Vec<u8>> {
let Some(mut reader) = reader else {
push_warning(
warnings,
source_path,
code,
"Skipped OneNote blob content: missing data",
);
return None;
};
let mut bytes = Vec::new();
if let Err(error) = reader.read_to_end(&mut bytes) {
push_warning(
warnings,
source_path,
code,
&format!("Skipped OneNote blob content: {error}"),
);
return None;
}
Some(bytes)
}
fn image_mime(image: &Image, file_name: &str, detected_mime: Option<&'static str>) -> String {
image
.extension()
.map(mime_from_extension)
.or(detected_mime)
.unwrap_or_else(|| mime_from_path(file_name))
.to_string()
}
fn blob_id(bytes: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(bytes);
hex::encode(hasher.finalize())
}
fn sanitize_file_name(value: &str, fallback: &str) -> String {
let name = value
.split(['/', '\\', '\0'])
.filter(|part| !part.is_empty() && *part != "." && *part != "..")
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string();
if name.is_empty() { fallback.to_string() } else { name }
}
fn mime_from_path(path: &str) -> &'static str {
match path.rsplit('.').next().unwrap_or("").to_lowercase().as_str() {
"png" => "image/png",
"jpg" | "jpeg" => "image/jpeg",
"gif" => "image/gif",
"webp" => "image/webp",
"bmp" => "image/bmp",
"svg" => "image/svg+xml",
"pdf" => "application/pdf",
"txt" => "text/plain",
"md" | "markdown" => "text/markdown",
"csv" => "text/csv",
"zip" => "application/zip",
"doc" => "application/msword",
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"xls" => "application/vnd.ms-excel",
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"ppt" => "application/vnd.ms-powerpoint",
"pptx" => "application/vnd.openxmlformats-officedocument.presentationml.presentation",
_ => "application/octet-stream",
}
}
fn mime_from_extension(extension: &str) -> &'static str {
mime_from_path(extension)
}
fn push_warning(warnings: &mut Vec<ImportWarning>, source_path: &str, code: &str, message: &str) {
warnings.push(ImportWarning {
code: code.to_string(),
source_path: Some(source_path.to_string()),
message: message.to_string(),
});
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
fn markdown_table_cell_collapses_multiline_content() {
assert_eq!(
markdown_table_cell("first line\nsecond | line\n\n third line "),
"first line<br />second \\| line<br />third line"
);
}
#[test]
fn remove_blocks_with_source_ids_removes_matching_nested_blocks() {
let mut snapshot = json!({
"children": [
{
"flavour": "affine:paragraph",
"props": {},
"children": [
{
"flavour": "affine:image",
"props": { "sourceId": "skipped" },
"children": []
},
{
"flavour": "affine:image",
"props": { "sourceId": "kept" },
"children": []
}
]
}
]
});
let source_ids = HashSet::from(["skipped".to_string()]);
remove_blocks_with_source_ids(&mut snapshot, &source_ids);
assert_eq!(snapshot["children"][0]["children"].as_array().unwrap().len(), 1);
assert_eq!(snapshot["children"][0]["children"][0]["props"]["sourceId"], "kept");
}
}
@@ -0,0 +1,171 @@
use onenote_parser::{
contents::{ParagraphStyling, RichText},
property::common::ColorRef,
};
pub(super) fn rich_text_markdown(text: &RichText) -> String {
let mut markdown = styled_text_markdown(text);
let trimmed = markdown.trim();
if trimmed.is_empty() {
return String::new();
}
if let Some(prefix) = heading_prefix(text, trimmed) {
markdown = format!("{prefix} {}", trimmed.trim_start_matches('#').trim_start());
}
markdown
}
fn styled_text_markdown(text: &RichText) -> String {
let chars = text.text().chars().collect::<Vec<_>>();
let styles = text.text_run_formatting();
if chars.is_empty() {
return String::new();
}
if styles.is_empty() {
return chars.into_iter().collect();
}
let mut output = String::new();
let mut start = 0usize;
let mut pending_link: Option<String> = None;
for (index, style) in styles.iter().enumerate() {
let end = text
.text_run_indices()
.get(index)
.map(|end| (*end as usize).min(chars.len()))
.unwrap_or(chars.len());
if end < start {
continue;
}
let segment = chars[start..end].iter().collect::<String>();
start = end;
if segment.is_empty() {
continue;
}
if style.hidden() {
if let Some(url) = extract_hyperlink_url(&segment) {
pending_link = Some(url);
}
continue;
}
output.push_str(&format_segment(&segment, style, &mut pending_link));
}
if start < chars.len() {
output.push_str(&chars[start..].iter().collect::<String>());
}
output
}
fn heading_prefix(text: &RichText, value: &str) -> Option<&'static str> {
if value.chars().count() > 120 || value.lines().count() > 1 {
return None;
}
let font_size = text
.text_run_formatting()
.iter()
.filter(|style| !style.hidden())
.filter_map(ParagraphStyling::font_size)
.max()
.or_else(|| text.paragraph_style().font_size());
match font_size {
Some(size) if size >= 28 => Some("##"),
Some(size) if size >= 24 => Some("###"),
_ => None,
}
}
fn format_segment(segment: &str, style: &ParagraphStyling, pending_link: &mut Option<String>) -> String {
let mut value = escape_html(segment);
if segment.trim().is_empty() {
return value;
}
if style.bold() && should_keep_inline_style(segment) {
value = wrap_inline(&value, "<strong>", "</strong>");
}
if style.italic() && should_keep_inline_style(segment) {
value = wrap_inline(&value, "<em>", "</em>");
}
if style.strikethrough() {
value = wrap_inline(&value, "<del>", "</del>");
}
if style.underline() {
value = wrap_inline(&value, "<u>", "</u>");
}
if let Some(color) = style.font_color().and_then(hex_color) {
value = wrap_inline(&value, &format!("<span style=\"color: {color}\">"), "</span>");
}
if let Some(color) = style.highlight().and_then(hex_color) {
value = wrap_inline(&value, &format!("<span style=\"color: {color}\">"), "</span>");
}
if style.hyperlink_protected()
&& let Some(url) = pending_link.take()
{
return wrap_inline(&value, &format!("<a href=\"{}\">", escape_html_attr(&url)), "</a>");
}
value
}
fn should_keep_inline_style(value: &str) -> bool {
value.chars().any(|ch| ch.is_alphanumeric())
}
fn wrap_inline(value: &str, open: &str, close: &str) -> String {
let leading_len = value.len() - value.trim_start().len();
let trailing_len = value.len() - value.trim_end().len();
let (leading, rest) = value.split_at(leading_len);
let (middle, trailing) = rest.split_at(rest.len().saturating_sub(trailing_len));
if middle.is_empty() {
value.to_string()
} else {
format!("{leading}{open}{middle}{close}{trailing}")
}
}
fn hex_color(color: ColorRef) -> Option<String> {
match color {
ColorRef::Auto => None,
ColorRef::Manual { r, g, b } => Some(format!("#{r:02x}{g:02x}{b:02x}")),
}
}
fn escape_html(value: &str) -> String {
let mut output = String::with_capacity(value.len());
for ch in value.chars() {
match ch {
'&' => output.push_str("&amp;"),
'<' => output.push_str("&lt;"),
'>' => output.push_str("&gt;"),
_ => output.push(ch),
}
}
output
}
fn escape_html_attr(value: &str) -> String {
let mut output = String::with_capacity(value.len());
for ch in value.chars() {
match ch {
'&' => output.push_str("&amp;"),
'<' => output.push_str("&lt;"),
'>' => output.push_str("&gt;"),
'"' => output.push_str("&quot;"),
_ => output.push(ch),
}
}
output
}
fn extract_hyperlink_url(value: &str) -> Option<String> {
let marker = "HYPERLINK";
let start = value.find(marker)? + marker.len();
let rest = value[start..].trim();
if let Some(quoted) = rest.strip_prefix('"') {
let end = quoted.find('"')?;
return Some(quoted[..end].to_string());
}
rest.split_whitespace().next().map(ToString::to_string)
}
@@ -0,0 +1,444 @@
use std::{
collections::{HashSet, hash_map::DefaultHasher},
fs,
hash::{Hash, Hasher},
path::{Path, PathBuf},
};
mod content;
use affine_importer::{
FolderHierarchyDelta, ImportBatch, ImportBatchLimits, ImportCursor, ImportError, ImportOptions, ImportProgress,
ImportProvider, ImportResult, ImportSource, ImportWarning, ImportedAsset, ImportedDocMeta, ImportedDocSnapshot,
};
use content::{page_markdown, remove_blocks_with_source_ids, rewrite_non_image_assets_to_attachments};
use onenote_parser::{
Parser,
notebook::Notebook,
page::Page,
section::{Section, SectionEntry, SectionGroup},
};
use typed_path::TypedPath;
pub struct OneNoteImportProvider;
impl OneNoteImportProvider {
pub fn new() -> Self {
Self
}
}
impl ImportProvider for OneNoteImportProvider {
fn format(&self) -> &'static str {
"oneNote"
}
fn create_cursor(
&self,
source: ImportSource,
_options: ImportOptions,
limits: ImportBatchLimits,
) -> ImportResult<Box<dyn ImportCursor>> {
let mut planner = OneNotePlanner {
docs: Vec::new(),
folders: Vec::new(),
warnings: Vec::new(),
};
planner.read_source(source)?;
Ok(Box::new(OneNoteImportCursor {
docs: planner.docs,
folders: planner.folders,
warnings: planner.warnings,
next_asset_doc: 0,
next_asset_index: 0,
next_doc: 0,
emitted_final: false,
skipped_blob_ids: HashSet::new(),
limits,
}))
}
}
struct OneNoteDoc {
id: String,
title: String,
markdown: String,
assets: Vec<ImportedAsset>,
}
struct OneNotePlanner {
docs: Vec<OneNoteDoc>,
folders: Vec<FolderHierarchyDelta>,
warnings: Vec<ImportWarning>,
}
impl OneNotePlanner {
fn read_source(&mut self, source: ImportSource) -> ImportResult<()> {
match source {
ImportSource::FilePath(path) => self.read_file(&path),
ImportSource::DirectoryPath(path) => self.read_directory(&path),
}
}
fn read_file(&mut self, path: &Path) -> ImportResult<()> {
let parser = Parser::new();
match extension(path).as_deref() {
Some("one") => {
let source_path = path.to_string_lossy();
let section = parser
.parse_section(TypedPath::derive(source_path.as_ref()))
.map_err(import_error)?;
self.add_section(&section, None);
Ok(())
}
Some("onetoc2") => {
let source_path = path.to_string_lossy();
let notebook = parser
.parse_notebook(TypedPath::derive(source_path.as_ref()))
.map_err(import_error)?;
self.add_notebook(&notebook, notebook_name(path));
Ok(())
}
Some("onepkg") => {
let source_path = path.to_string_lossy();
let notebook = parser
.parse_package(TypedPath::derive(source_path.as_ref()))
.map_err(import_error)?;
self.add_notebook(&notebook, notebook_name(path));
Ok(())
}
_ => Err(ImportError::InvalidSource(format!(
"unsupported OneNote source: {}",
path.to_string_lossy()
))),
}
}
fn read_directory(&mut self, path: &Path) -> ImportResult<()> {
let toc = find_first_onetoc2(path)?.ok_or_else(|| {
ImportError::InvalidSource(format!(
"directory contains no .onetoc2 file: {}",
path.to_string_lossy()
))
})?;
let parser = Parser::new();
let toc_path = toc.to_string_lossy();
let notebook = parser
.parse_notebook(TypedPath::derive(toc_path.as_ref()))
.map_err(import_error)?;
self.add_notebook(&notebook, notebook_name(path));
Ok(())
}
fn add_notebook(&mut self, notebook: &Notebook, name: String) {
let root = sanitize_path_part(&name);
self.folders.push(FolderHierarchyDelta {
path: root.clone(),
name,
parent_path: None,
page_id: None,
icon: None,
});
for entry in notebook.entries() {
self.add_section_entry(entry, Some(root.clone()));
}
}
fn add_section_entry(&mut self, entry: &SectionEntry, parent_path: Option<String>) {
match entry {
SectionEntry::Section(section) => self.add_section(section, parent_path),
SectionEntry::SectionGroup(group) => self.add_section_group(group, parent_path),
}
}
fn add_section_group(&mut self, group: &SectionGroup, parent_path: Option<String>) {
let path = join_import_path(parent_path.as_deref(), group.display_name());
self.folders.push(FolderHierarchyDelta {
path: path.clone(),
name: group.display_name().to_string(),
parent_path,
page_id: None,
icon: None,
});
for entry in group.entries() {
self.add_section_entry(entry, Some(path.clone()));
}
}
fn add_section(&mut self, section: &Section, parent_path: Option<String>) {
let path = join_import_path(parent_path.as_deref(), section.display_name());
self.folders.push(FolderHierarchyDelta {
path: path.clone(),
name: section.display_name().to_string(),
parent_path,
page_id: None,
icon: None,
});
let mut page_index = 0;
for series in section.page_series() {
for page in series.pages() {
self.add_page(page, &path, page_index);
page_index += 1;
}
}
}
fn add_page(&mut self, page: &Page, parent_path: &str, page_index: usize) {
let title = page
.title_text()
.map(str::trim)
.filter(|title| !title.is_empty())
.unwrap_or("Untitled")
.to_string();
let source_path = join_import_path(Some(parent_path), &title);
let id = doc_id(page.link_target_id(), &title, parent_path, page_index);
let mut assets = Vec::new();
let markdown = page_markdown(page, &mut assets, &mut self.warnings, &source_path);
self.folders.push(FolderHierarchyDelta {
path: source_path.clone(),
name: title.clone(),
parent_path: Some(parent_path.to_string()),
page_id: Some(id.clone()),
icon: None,
});
self.docs.push(OneNoteDoc {
id,
title,
markdown,
assets,
});
}
}
struct OneNoteImportCursor {
docs: Vec<OneNoteDoc>,
folders: Vec<FolderHierarchyDelta>,
warnings: Vec<ImportWarning>,
next_asset_doc: usize,
next_asset_index: usize,
next_doc: usize,
emitted_final: bool,
skipped_blob_ids: HashSet<String>,
limits: ImportBatchLimits,
}
impl ImportCursor for OneNoteImportCursor {
fn total(&self) -> usize {
self.docs.len()
}
fn next_batch(&mut self) -> ImportResult<Option<ImportBatch>> {
if self.next_doc >= self.docs.len() && self.emitted_final {
return Ok(None);
}
if let Some(batch) = self.next_asset_batch() {
return Ok(Some(batch));
}
let end = (self.next_doc + self.limits.max_docs.max(1)).min(self.docs.len());
let mut batch = empty_batch(self.docs.len());
for doc in &self.docs[self.next_doc..end] {
let mut snapshot = affine_doc_loader::build_doc_snapshot(&doc.title, &doc.markdown, &doc.id)?;
rewrite_non_image_assets_to_attachments(&mut snapshot, &doc.assets);
remove_blocks_with_source_ids(&mut snapshot, &self.skipped_blob_ids);
batch.docs.push(ImportedDocSnapshot {
id: doc.id.clone(),
snapshot,
meta: Some(ImportedDocMeta {
title: Some(doc.title.clone()),
create_date: None,
updated_date: None,
tags: None,
favorite: None,
trash: None,
}),
});
}
self.next_doc = end;
if self.next_doc >= self.docs.len() {
batch.folders = std::mem::take(&mut self.folders);
batch.warnings = std::mem::take(&mut self.warnings);
self.emitted_final = true;
}
batch.progress = ImportProgress {
completed: self.next_doc,
total: self.docs.len(),
};
batch.done = self.emitted_final;
Ok(Some(batch))
}
}
impl OneNoteImportCursor {
fn next_asset_batch(&mut self) -> Option<ImportBatch> {
let max_blobs = self.limits.max_blobs.max(1);
let max_blob_bytes = self.limits.max_blob_bytes;
let mut batch = empty_batch(self.docs.len());
let mut bytes_in_batch = 0u64;
while self.next_asset_doc < self.docs.len() && batch.blobs.len() < max_blobs {
let doc = &self.docs[self.next_asset_doc];
if self.next_asset_index >= doc.assets.len() {
self.next_asset_doc += 1;
self.next_asset_index = 0;
continue;
}
let asset = &doc.assets[self.next_asset_index];
self.next_asset_index += 1;
let asset_size = asset.bytes.len() as u64;
if asset_size > max_blob_bytes {
self.skipped_blob_ids.insert(asset.blob_id.clone());
batch.warnings.push(ImportWarning {
code: "blob_too_large".to_string(),
source_path: Some(asset.source_path.clone()),
message: format!("Skipped {}: blob is larger than batch limit", asset.source_path),
});
continue;
}
if !batch.blobs.is_empty() && bytes_in_batch + asset_size > max_blob_bytes {
self.next_asset_index -= 1;
break;
}
bytes_in_batch += asset_size;
batch.blobs.push(asset.clone());
}
if batch.blobs.is_empty() && batch.warnings.is_empty() {
return None;
}
batch.progress = ImportProgress {
completed: self.next_doc,
total: self.docs.len(),
};
Some(batch)
}
}
fn empty_batch(total: usize) -> ImportBatch {
ImportBatch {
docs: Vec::new(),
blobs: Vec::new(),
folders: Vec::new(),
tags: Vec::new(),
icons: Vec::new(),
warnings: Vec::new(),
progress: ImportProgress { completed: 0, total },
entry_id: None,
is_workspace_file: false,
done: false,
}
}
fn find_first_onetoc2(root: &Path) -> ImportResult<Option<PathBuf>> {
for entry in fs::read_dir(root)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
if let Some(found) = find_first_onetoc2(&path)? {
return Ok(Some(found));
}
} else if extension(&path).as_deref() == Some("onetoc2") {
return Ok(Some(path));
}
}
Ok(None)
}
fn import_error(error: onenote_parser::errors::Error) -> ImportError {
ImportError::InvalidSource(error.to_string())
}
fn notebook_name(path: &Path) -> String {
path
.file_stem()
.or_else(|| path.file_name())
.map(|name| name.to_string_lossy().to_string())
.filter(|name| !name.is_empty())
.unwrap_or_else(|| "OneNote".to_string())
}
fn extension(path: &Path) -> Option<String> {
path.extension().map(|ext| ext.to_string_lossy().to_lowercase())
}
fn join_import_path(parent: Option<&str>, name: &str) -> String {
let name = sanitize_path_part(name);
parent
.filter(|parent| !parent.is_empty())
.map(|parent| format!("{parent}/{name}"))
.unwrap_or(name)
}
fn sanitize_path_part(value: &str) -> String {
let name = value
.split(['/', '\\'])
.filter(|part| !part.is_empty())
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string();
if name.is_empty() { "Untitled".to_string() } else { name }
}
fn doc_id(raw_id: &str, title: &str, parent_path: &str, page_index: usize) -> String {
let mut id = raw_id
.chars()
.filter(|ch| ch.is_ascii_alphanumeric() || *ch == '-' || *ch == '_')
.collect::<String>();
if id.is_empty() {
let slug = title
.chars()
.filter(|ch| ch.is_ascii_alphanumeric())
.take(32)
.collect::<String>();
let mut hasher = DefaultHasher::new();
title.hash(&mut hasher);
parent_path.hash(&mut hasher);
page_index.hash(&mut hasher);
let hash = format!("{:x}", hasher.finish());
id = if slug.is_empty() {
hash
} else {
format!("{slug}-{hash}")
};
}
format!("onenote-{id}")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sanitize_path_part_removes_path_separators() {
assert_eq!(sanitize_path_part("Group/Section\\Page"), "Group Section Page");
assert_eq!(sanitize_path_part("///"), "Untitled");
}
#[test]
fn join_import_path_sanitizes_path_parts() {
assert_eq!(
join_import_path(Some("Notebook/Section"), "Page/Subpage\\Draft"),
"Notebook/Section/Page Subpage Draft"
);
}
#[test]
fn doc_id_keeps_ascii_ids_and_hashes_non_ascii_titles() {
assert_eq!(doc_id("{abc-123}", "Ignored", "Notebook/Section", 0), "onenote-abc-123");
let id = doc_id("", "中文标题", "Notebook/Section", 0);
assert!(id.starts_with("onenote-"));
assert!(id.len() > "onenote-".len());
}
#[test]
fn doc_id_fallback_distinguishes_duplicate_titles() {
let first = doc_id("", "Untitled", "Notebook/Section", 0);
let second = doc_id("", "Untitled", "Notebook/Section", 1);
assert_ne!(first, second);
}
}
@@ -2,13 +2,13 @@ use std::{
collections::HashMap,
path::PathBuf,
sync::{
Mutex,
Arc, Mutex,
atomic::{AtomicU64, Ordering},
},
};
use affine_importer::{
ImportBatchLimits, ImportFormat, ImportOptions, ImportSession, ImportSessionOptions, ImportSessionSource,
ImportBatchLimits, ImportOptions, ImportRegistry, ImportSession, ImportSessionOptions, ImportSource,
};
use napi::{Status, bindgen_prelude::*};
use napi_derive::napi;
@@ -16,6 +16,11 @@ use once_cell::sync::Lazy;
static IMPORT_SESSIONS: Lazy<Mutex<HashMap<String, ImportSession>>> = Lazy::new(|| Mutex::new(HashMap::new()));
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
static IMPORT_REGISTRY: Lazy<Arc<ImportRegistry>> = Lazy::new(|| {
let mut registry = ImportRegistry::with_builtin();
registry.register(super::OneNoteImportProvider::new());
Arc::new(registry)
});
#[napi(object)]
pub struct CreateImportSessionOptions {
@@ -37,25 +42,10 @@ pub struct CreateImportBatchLimits {
pub max_blob_bytes: Option<i64>,
}
fn parse_format(format: &str) -> Result<ImportFormat> {
match format {
"markdownZip" => Ok(ImportFormat::MarkdownZip),
"notionZip" => Ok(ImportFormat::NotionZip),
"notionMarkdownZip" => Ok(ImportFormat::NotionMarkdownZip),
"notionHtmlZip" => Ok(ImportFormat::NotionHtmlZip),
"obsidian" => Ok(ImportFormat::Obsidian),
"bearZip" => Ok(ImportFormat::BearZip),
_ => Err(Error::new(
Status::InvalidArg,
format!("unsupported import format: {format}"),
)),
}
}
fn map_import_error(error: affine_importer::ImportError) -> Error {
let status = match error {
affine_importer::ImportError::Cancelled => Status::Cancelled,
affine_importer::ImportError::UnsupportedFormat | affine_importer::ImportError::InvalidSource(_) => {
affine_importer::ImportError::UnsupportedFormat(_) | affine_importer::ImportError::InvalidSource(_) => {
Status::InvalidArg
}
affine_importer::ImportError::Zip(_)
@@ -67,10 +57,9 @@ fn map_import_error(error: affine_importer::ImportError) -> Error {
#[napi]
pub fn create_import_session(options: CreateImportSessionOptions) -> Result<String> {
let format = parse_format(&options.format)?;
let source = match options.source.kind.as_str() {
"filePath" => ImportSessionSource::FilePath(PathBuf::from(options.source.path)),
"directoryPath" => ImportSessionSource::DirectoryPath(PathBuf::from(options.source.path)),
"filePath" => ImportSource::FilePath(PathBuf::from(options.source.path)),
"directoryPath" => ImportSource::DirectoryPath(PathBuf::from(options.source.path)),
kind => {
return Err(Error::new(
Status::InvalidArg,
@@ -93,12 +82,15 @@ pub fn create_import_session(options: CreateImportSessionOptions) -> Result<Stri
batch_limits.max_blob_bytes = max_blob_bytes as u64;
}
}
let session = ImportSession::create(ImportSessionOptions {
format,
source,
import_options: ImportOptions::default(),
batch_limits,
})
let session = ImportSession::create(
ImportSessionOptions {
format: options.format,
source,
import_options: ImportOptions::default(),
batch_limits,
},
IMPORT_REGISTRY.clone(),
)
.map_err(map_import_error)?;
let id = format!("import-session-{}", NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed));
IMPORT_SESSIONS
@@ -267,4 +259,21 @@ mod tests {
assert!(result.is_err());
let _ = std::fs::remove_file(path);
}
#[test]
fn import_session_routes_onenote_format_to_provider() {
let path = archive_path(&[("entry.md", b"entry")]);
let result = create_import_session(CreateImportSessionOptions {
format: "oneNote".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
});
let error = result.unwrap_err().to_string();
assert!(error.contains("unsupported OneNote source"));
let _ = std::fs::remove_file(path);
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod hashcash;
mod import_session;
mod import;
#[cfg(not(any(target_os = "android", target_os = "ios")))]
pub mod preview;
@@ -11,4 +11,4 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
pub use affine_media_capture::*;
pub use affine_nbstore::*;
pub use affine_sqlite_v1::*;
pub use import_session::*;
pub use import::*;