feat(core): import progress & perf (#15197)

#### PR Dependency Tree


* **PR #15197** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a new import pipeline with “plan then commit” batch handling for
Markdown, Notion HTML, Obsidian, and Bear backups.
* Enabled native import sessions with progress, cancellation, and
batch-by-batch committing (including assets, folders, icons, and tags).
  * Added web preflight limits for ZIP and multi-file imports.
* **Bug Fixes**
* Improved import error/warning reporting and continued processing when
some items fail.
* Strengthened snapshot-based file/directory picking to preserve paths.
* **Chores**
  * Updated project packaging/configuration for the new import workflow.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-06 01:29:24 +08:00
committed by GitHub
parent 477015f064
commit 8d72e4dc29
106 changed files with 3329 additions and 11690 deletions
@@ -2,7 +2,11 @@ import { readFileSync } from 'node:fs';
import { basename, resolve } from 'node:path';
import {
BearTransformer,
commitImportBatchToWorkspace,
type ImportBatch,
MarkdownTransformer,
NotionHtmlTransformer,
ObsidianTransformer,
} from '@blocksuite/affine/widgets/linked-doc';
import {
@@ -84,6 +88,23 @@ function zipFixture(entries: Record<string, string | Uint8Array>) {
return new Blob([buffer], { type: 'application/zip' });
}
async function commitPlannedImport<T extends { batch: ImportBatch }>(
collection: TestWorkspace,
schema: Schema,
planned: T
) {
const committed = await commitImportBatchToWorkspace(
collection,
schema,
planned.batch
);
return {
...planned,
...committed,
docIds: committed.docIds,
};
}
function exportSnapshot(doc: Store): DocSnapshot {
const job = doc.getTransformer([
docLinkBaseURLMiddleware(doc.workspace.id),
@@ -203,6 +224,12 @@ function snapshotDocByTitle(
return simplifyBlockForSnapshot(exportSnapshot(doc!).blocks, titleById);
}
function titleMap(collection: TestWorkspace) {
return new Map(
collection.meta.docMetas.map(meta => [meta.id, meta.title ?? '<untitled>'])
);
}
function collectSimplifiedDeltas(
block: Record<string, unknown>
): Record<string, unknown>[] {
@@ -218,6 +245,27 @@ function collectSimplifiedDeltas(
return [...deltas, ...childDeltas];
}
function collectSnapshotDeltas(
block: BlockSnapshot
): DeltaInsert<AffineTextAttributes>[] {
const text = block.props.text as
| { delta?: DeltaInsert<AffineTextAttributes>[] }
| undefined;
return [
...(text?.delta ?? []),
...(block.children ?? []).flatMap(child => collectSnapshotDeltas(child)),
];
}
function folderChild(
folder: { children: Map<string, unknown> } | undefined,
name: string
) {
return folder?.children.get(name) as
| { children: Map<string, unknown>; pageId?: string; icon?: unknown }
| undefined;
}
describe('snapshot to markdown', () => {
test('code', async () => {
const blockSnapshot: BlockSnapshot = {
@@ -377,13 +425,16 @@ Hello world
'# Nested Page\nNested body',
});
const { docIds, folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
const { docIds, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
});
})
);
expect(docIds).toHaveLength(2);
expect(
@@ -431,13 +482,16 @@ Hello world
'# SDK架构\nNested body',
});
const { folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
const { folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
});
})
);
const [rootFolder] = [...(folderHierarchy?.children.values() ?? [])];
expect(rootFolder?.name).toBe('Export');
@@ -463,12 +517,16 @@ Hello world
'---\ntitle: Frontmatter Title\n---\nBody',
});
const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({
const { docIds } = await commitPlannedImport(
collection,
schema,
imported,
extensions: testStoreExtensions,
});
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(1);
expect(collection.meta.getDocMeta(docIds[0])?.title).toBe(
@@ -491,12 +549,16 @@ Hello world
'test/2.md': 'target page',
});
const { docIds } = await MarkdownTransformer.importMarkdownZip({
const { docIds } = await commitPlannedImport(
collection,
schema,
imported,
extensions: testStoreExtensions,
});
await MarkdownTransformer.planMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(2);
const titleById = new Map(
@@ -527,6 +589,88 @@ Hello world
});
});
test('imports markdown zip assets, nested zip, CJK paths, and duplicate names', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const imported = zipFixture({
'入口.md':
'![logo](./assets/logo.png)\n[同名](./folder/duplicate.md)\n![archive](./nested.zip)',
'assets/logo.png': new Uint8Array([137, 80, 78, 71]),
'folder/duplicate.md': 'folder duplicate',
'other/duplicate.md': 'other duplicate',
'nested.zip': zipBytes({
'ignored.md': 'nested markdown should stay an attachment',
}),
});
const { docIds, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(3);
expect(
collection.meta.docMetas
.map(meta => meta.title)
.sort((a, b) => (a ?? '').localeCompare(b ?? ''))
).toEqual(['duplicate', 'duplicate', '入口']);
const titles = titleMap(collection);
const entry = snapshotDocByTitle(collection, '入口', titles);
expect(JSON.stringify(entry)).toContain('"sourceId":"<asset>"');
const entrySnapshot = exportSnapshot(
collection
.getDoc(
collection.meta.docMetas.find(meta => meta.title === '入口')!.id
)!
.getStore({
id: collection.meta.docMetas.find(meta => meta.title === '入口')!.id,
})
);
const linkedPageDelta = collectSnapshotDeltas(entrySnapshot.blocks).find(
delta => delta.attributes?.reference?.type === 'LinkedPage'
);
const linkedPageId =
linkedPageDelta?.attributes?.reference?.type === 'LinkedPage'
? linkedPageDelta.attributes.reference.pageId
: undefined;
expect(linkedPageId).toBeTruthy();
expect(
JSON.stringify(
snapshotDocByTitle(
collection,
'duplicate',
new Map([[linkedPageId!, 'duplicate']])
)
)
).toContain('folder duplicate');
expect(collectSimplifiedDeltas(entry)).toContainEqual({
insert: ' ',
reference: {
type: 'LinkedPage',
page: 'duplicate',
title: '同名',
},
});
expect(JSON.stringify(entry).match(/"sourceId":"<asset>"/g)).toHaveLength(
2
);
expect(folderHierarchy?.children.has('folder')).toBe(true);
expect(folderHierarchy?.children.has('other')).toBe(true);
expect(
collection.meta.docMetas.some(meta => meta.title === 'ignored')
).toBe(false);
});
test('imports notion markdown zip relative doc links as linked pages', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
@@ -540,12 +684,16 @@ Hello world
'# Target\ntarget page',
});
const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({
const { docIds } = await commitPlannedImport(
collection,
schema,
imported,
extensions: testStoreExtensions,
});
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(2);
const titleById = new Map(
@@ -587,13 +735,16 @@ Hello world
}),
});
const { docIds, folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
const { docIds, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
});
})
);
expect(docIds).toHaveLength(4);
const titleById = new Map(
@@ -646,16 +797,20 @@ Hello world
'vault/archive.zip'
);
const { docIds } = await ObsidianTransformer.importObsidianVault({
const { docIds } = await commitPlannedImport(
collection,
schema,
importedFiles: [
markdownFixture('entry.md'),
markdownFixture('linked.md'),
attachment,
],
extensions: testStoreExtensions,
});
await ObsidianTransformer.planObsidianVault({
collection,
schema,
importedFiles: [
markdownFixture('entry.md'),
markdownFixture('linked.md'),
attachment,
],
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(2);
const titleById = new Map(
@@ -673,6 +828,147 @@ Hello world
}).toMatchSnapshot();
});
test('imports notion html zip golden baseline', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const imported = zipFixture({
'Export/index.html': '<html><body>workspace index</body></html>',
'Export/Project.html': `
<html>
<body>
<div class="page-header-icon undefined"><span class="icon">✅</span></div>
<div class="page-body">
<p id="11111111-1111-1111-1111-111111111111" class="">Project body</p>
<img id="22222222-2222-2222-2222-222222222222" src="assets/logo.png" />
</div>
</body>
</html>
`,
'Export/Project/Nested.html': `
<html>
<body>
<div class="page-body"><p id="33333333-3333-3333-3333-333333333333" class="">Nested body</p></div>
</body>
</html>
`,
'Export/assets/logo.png': new Uint8Array([137, 80, 78, 71]),
});
const result = await commitPlannedImport(
collection,
schema,
await NotionHtmlTransformer.planNotionHtmlZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(result.isWorkspaceFile).toBe(true);
expect(result.hasMarkdown).toBe(false);
expect(result.pageIds).toHaveLength(2);
expect(collection.meta.docMetas.map(meta => meta.title)).toEqual(['', '']);
const titles = titleMap(collection);
const importedSnapshots = result.pageIds.map(pageId =>
simplifyBlockForSnapshot(
exportSnapshot(collection.getDoc(pageId)!.getStore({ id: pageId }))
.blocks,
titles
)
);
const projectSnapshot = importedSnapshots.find(snapshot =>
JSON.stringify(snapshot).includes('Project body')
);
expect(projectSnapshot).toBeTruthy();
expect(JSON.stringify(projectSnapshot)).toContain('Project body');
expect(JSON.stringify(projectSnapshot)).toContain('"sourceId":"<asset>"');
const exportFolder = folderChild(result.folderHierarchy, 'Export');
const projectNode = folderChild(exportFolder, 'Project');
expect(result.pageIds).toContain(projectNode?.pageId);
expect(projectNode?.icon).toEqual({ type: 'emoji', content: '✅' });
expect(result.pageIds).toContain(
folderChild(projectNode, 'Nested')?.pageId
);
});
test('imports bear backup golden baseline', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const imported = zipFixture({
'Notes/Idea.textbundle/text.md': [
'# Bear Title',
'',
'![photo](assets/photo.png)',
'',
'==🟢green highlight==',
'',
'#work/project',
'#Blue Tag#',
].join('\n'),
'Notes/Idea.textbundle/info.json': JSON.stringify({
'net.shinyfrog.bear': {
creationDate: '2024-01-02T03:04:05.000Z',
modificationDate: '2024-01-03T03:04:05.000Z',
},
}),
'Notes/Idea.textbundle/assets/photo.png': new Uint8Array([
137, 80, 78, 71,
]),
});
const { docIds, tags, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await BearTransformer.planBearBackup({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(1);
const meta = collection.meta.getDocMeta(docIds[0]);
expect(meta?.title).toBe('Bear Title');
expect(meta?.createDate).toBe(Date.parse('2024-01-02T03:04:05.000Z'));
expect(meta?.updatedDate).toBe(Date.parse('2024-01-03T03:04:05.000Z'));
expect([...tags.keys()]).toEqual(['Blue Tag', 'work/project']);
const titles = titleMap(collection);
const snapshot = snapshotDocByTitle(collection, 'Bear Title', titles);
expect(JSON.stringify(snapshot)).toContain('"sourceId":"<asset>"');
expect(JSON.stringify(snapshot)).toContain('green highlight');
const blueTag = folderChild(folderHierarchy, 'Blue Tag');
expect([
...(
(blueTag?.children as Map<string, unknown> | undefined) ?? new Map()
).values(),
]).toEqual(
expect.arrayContaining([expect.objectContaining({ pageId: docIds[0] })])
);
const project = folderChild(
folderChild(folderHierarchy, 'work'),
'project'
);
expect([
...(
(project?.children as Map<string, unknown> | undefined) ?? new Map()
).values(),
]).toEqual(
expect.arrayContaining([expect.objectContaining({ pageId: docIds[0] })])
);
});
test('paragraph', async () => {
const blockSnapshot: BlockSnapshot = {
type: 'block',
@@ -142,6 +142,55 @@ type AcceptTypes =
| 'Docx'
| 'MindMap';
type OpenFileOptions = {
fileSystemAccess?: boolean;
snapshot?: boolean;
};
function copyToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
export async function snapshotFile(file: File, relativePath?: string) {
let bytes: Uint8Array | null = null;
for (let attempt = 0; attempt < 3; attempt++) {
try {
bytes = new Uint8Array(await file.arrayBuffer());
break;
} catch (error) {
if (!isNotReadableError(error) || attempt === 2) {
throw error;
}
await sleep(100 * (attempt + 1));
}
}
if (!bytes) {
throw new DOMException('File could not be read', 'NotReadableError');
}
const snapshot = new File([copyToArrayBuffer(bytes)], file.name, {
type: file.type,
lastModified: file.lastModified,
});
const path = relativePath ?? file.webkitRelativePath;
if (path) {
Object.defineProperty(snapshot, 'webkitRelativePath', {
value: path,
writable: false,
});
}
return snapshot;
}
export async function snapshotFiles(files: File[]) {
return Promise.all(files.map(file => snapshotFile(file)));
}
function canUseFileSystemAccessAPI(
api: 'showOpenFilePicker' | 'showDirectoryPicker'
) {
@@ -157,11 +206,21 @@ function canUseFileSystemAccessAPI(
);
}
function isAbortError(error: unknown) {
return error instanceof Error && error.name === 'AbortError';
}
function isNotReadableError(error: unknown) {
return error instanceof Error && error.name === 'NotReadableError';
}
export async function openFilesWith(
acceptType: AcceptTypes = 'Any',
multiple: boolean = true
multiple: boolean = true,
options?: OpenFileOptions
): Promise<File[] | null> {
const supportsFileSystemAccess =
options?.fileSystemAccess !== false &&
canUseFileSystemAccessAPI('showOpenFilePicker');
// If the File System Access API is supported…
@@ -180,15 +239,19 @@ export async function openFilesWith(
// Show the file picker, optionally allowing multiple files.
const handles = await window.showOpenFilePicker(pickerOpts);
return await Promise.all(handles.map(handle => handle.getFile()));
const files = await Promise.all(handles.map(handle => handle.getFile()));
return options?.snapshot ? await snapshotFiles(files) : files;
} catch (err) {
if (options?.snapshot && !isAbortError(err)) {
throw err;
}
console.error(err);
return null;
}
}
// Fallback if the File System Access API is not supported.
return new Promise(resolve => {
return new Promise((resolve, reject) => {
// Append a new `<input type="file" multiple? />` and hide it.
const input = document.createElement('input');
input.classList.add('affine-upload-input');
@@ -205,13 +268,27 @@ export async function openFilesWith(
document.body.append(input);
// The `change` event fires when the user interacts with the dialog.
input.addEventListener('change', () => {
// Remove the `<input type="file" multiple? />` again from the DOM.
input.remove();
const files = input.files ? Array.from(input.files) : null;
const finish = (result: File[] | null) => {
// Remove the `<input type="file" multiple? />` again from the DOM.
input.remove();
resolve(result);
};
resolve(input.files ? Array.from(input.files) : null);
if (files && options?.snapshot) {
snapshotFiles(files).then(finish, error => {
input.remove();
reject(error);
});
} else {
finish(files);
}
});
// The `cancel` event fires when the user cancels the dialog.
input.addEventListener('cancel', () => resolve(null));
input.addEventListener('cancel', () => {
input.remove();
resolve(null);
});
// Show the picker.
if ('showPicker' in HTMLInputElement.prototype) {
input.showPicker();
@@ -221,12 +298,18 @@ export async function openFilesWith(
});
}
export async function openDirectory(): Promise<File[] | null> {
export async function openDirectory(
options?: OpenFileOptions
): Promise<File[] | null> {
const supportsFileSystemAccess = canUseFileSystemAccessAPI(
'showDirectoryPicker'
);
if (supportsFileSystemAccess && window.showDirectoryPicker) {
if (
options?.fileSystemAccess !== false &&
supportsFileSystemAccess &&
window.showDirectoryPicker
) {
try {
const dirHandle = await window.showDirectoryPicker();
const files: File[] = [];
@@ -241,11 +324,15 @@ export async function openDirectory(): Promise<File[] | null> {
const fileHandle = handle as FileSystemFileHandle;
if (fileHandle.getFile) {
const file = await fileHandle.getFile();
Object.defineProperty(file, 'webkitRelativePath', {
value: relativePath,
writable: false,
});
files.push(file);
if (options?.snapshot) {
files.push(await snapshotFile(file, relativePath));
} else {
Object.defineProperty(file, 'webkitRelativePath', {
value: relativePath,
writable: false,
});
files.push(file);
}
}
} else if (handle.kind === 'directory') {
await readDirectory(
@@ -259,12 +346,15 @@ export async function openDirectory(): Promise<File[] | null> {
await readDirectory(dirHandle, '');
return files;
} catch (err) {
if (options?.snapshot && !isAbortError(err)) {
throw err;
}
console.error(err);
return null;
}
}
return new Promise(resolve => {
return new Promise((resolve, reject) => {
const input = document.createElement('input');
input.classList.add('affine-upload-input');
input.style.display = 'none';
@@ -276,11 +366,26 @@ export async function openDirectory(): Promise<File[] | null> {
document.body.append(input);
input.addEventListener('change', () => {
input.remove();
resolve(input.files ? Array.from(input.files) : null);
const files = input.files ? Array.from(input.files) : null;
const finish = (result: File[] | null) => {
input.remove();
resolve(result);
};
if (files && options?.snapshot) {
snapshotFiles(files).then(finish, error => {
input.remove();
reject(error);
});
} else {
finish(files);
}
});
input.addEventListener('cancel', () => resolve(null));
input.addEventListener('cancel', () => {
input.remove();
resolve(null);
});
if ('showPicker' in HTMLInputElement.prototype) {
input.showPicker();
@@ -2,15 +2,9 @@ import {
CloseIcon,
ExportToHTMLIcon,
ExportToMarkdownIcon,
HelpIcon,
NewIcon,
NotionIcon,
} from '@blocksuite/affine-components/icons';
import {
openDirectory,
openFilesWith,
openSingleFileWith,
} from '@blocksuite/affine-shared/utils';
import { openFilesWith } from '@blocksuite/affine-shared/utils';
import { WithDisposable } from '@blocksuite/global/lit';
import type { ExtensionType, Schema, Workspace } from '@blocksuite/store';
import { html, LitElement, type PropertyValues } from 'lit';
@@ -18,8 +12,6 @@ import { query, state } from 'lit/decorators.js';
import { HtmlTransformer } from '../transformers/html.js';
import { MarkdownTransformer } from '../transformers/markdown.js';
import { NotionHtmlTransformer } from '../transformers/notion-html.js';
import { ObsidianTransformer } from '../transformers/obsidian.js';
import { styles } from './styles.js';
export type OnSuccessHandler = (
@@ -59,114 +51,69 @@ export class ImportDoc extends WithDisposable(LitElement) {
}
private async _importHtml() {
const files = await openFilesWith('Html');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const needLoading = file.size > SHOW_LOADING_SIZE;
const fileName = file.name.split('.').slice(0, -1).join('.');
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await HtmlTransformer.importHTMLToDoc({
collection: this.collection,
schema: this.schema,
extensions: this.extensions,
html: text,
fileName,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
try {
const files = await openFilesWith('Html');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const needLoading = file.size > SHOW_LOADING_SIZE;
const fileName = file.name.split('.').slice(0, -1).join('.');
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await HtmlTransformer.importHTMLToDoc({
collection: this.collection,
schema: this.schema,
extensions: this.extensions,
html: text,
fileName,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
}
}
this._onImportSuccess(pageIds);
} catch (error) {
this._onFail(error);
}
this._onImportSuccess(pageIds);
}
private async _importMarkDown() {
const files = await openFilesWith('Markdown');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const fileName = file.name.split('.').slice(0, -1).join('.');
const needLoading = file.size > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await MarkdownTransformer.importMarkdownToDoc({
collection: this.collection,
schema: this.schema,
markdown: text,
fileName,
extensions: this.extensions,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
try {
const files = await openFilesWith('Markdown');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const fileName = file.name.split('.').slice(0, -1).join('.');
const needLoading = file.size > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await MarkdownTransformer.importMarkdownToDoc({
collection: this.collection,
schema: this.schema,
markdown: text,
fileName,
extensions: this.extensions,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
}
}
this._onImportSuccess(pageIds);
} catch (error) {
this._onFail(error);
}
this._onImportSuccess(pageIds);
}
private async _importNotion() {
const file = await openSingleFileWith('Zip');
if (!file) return;
const needLoading = file.size > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const { entryId, pageIds, isWorkspaceFile, hasMarkdown } =
await NotionHtmlTransformer.importNotionZip({
collection: this.collection,
schema: this.schema,
imported: file,
extensions: this.extensions,
});
needLoading && this.abortController.abort();
if (hasMarkdown) {
this._onFail(
'Importing markdown files from Notion is deprecated. Please export your Notion pages as HTML.'
);
return;
}
this._onImportSuccess(entryId ? [entryId] : [], {
isWorkspaceFile,
importedCount: pageIds.length,
});
}
private async _importObsidian() {
const files = await openDirectory();
if (!files || files.length === 0) return;
const needLoading =
files.reduce((acc, f) => acc + f.size, 0) > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const { docIds, docEmojis } = await ObsidianTransformer.importObsidianVault(
{
collection: this.collection,
schema: this.schema,
importedFiles: files,
extensions: this.extensions,
}
);
needLoading && this.abortController.abort();
this._onImportSuccess(docIds, { docEmojis });
}
private _onCloseClick(event: MouseEvent) {
@@ -174,8 +121,8 @@ export class ImportDoc extends WithDisposable(LitElement) {
this.abortController.abort();
}
private _onFail(message: string) {
this.onFail?.(message);
private _onFail(error: unknown) {
this.onFail?.(error instanceof Error ? error.message : String(error));
}
private _onImportSuccess(
@@ -213,14 +160,6 @@ export class ImportDoc extends WithDisposable(LitElement) {
window.removeEventListener('mousemove', this._onMouseMove);
}
private _openLearnImportLink(event: MouseEvent) {
event.stopPropagation();
window.open(
'https://affine.pro/blog/import-your-data-from-notion-into-affine',
'_blank'
);
}
override render() {
if (this._loading) {
return html`
@@ -276,30 +215,6 @@ export class ImportDoc extends WithDisposable(LitElement) {
</icon-button>
</div>
<div class="button-container">
<icon-button
class="button-item"
text="Notion"
@click="${this._importNotion}"
>
${NotionIcon}
<div
slot="suffix"
class="button-suffix"
@click="${this._openLearnImportLink}"
>
${HelpIcon}
<affine-tooltip>
Learn how to Import your Notion pages into AFFiNE.
</affine-tooltip>
</div>
</icon-button>
<icon-button
class="button-item"
text="Obsidian"
@click="${this._importObsidian}"
>
${ExportToMarkdownIcon}
</icon-button>
<icon-button class="button-item" text="Coming soon..." disabled>
${NewIcon}
</icon-button>
@@ -7,17 +7,30 @@ import {
} from '@blocksuite/affine-shared/adapters';
import { Container } from '@blocksuite/global/di';
import { sha } from '@blocksuite/global/utils';
import type { ExtensionType, Schema, Workspace } from '@blocksuite/store';
import type {
DocSnapshot,
ExtensionType,
Schema,
Workspace,
} from '@blocksuite/store';
import { extMimeMap, Transformer } from '@blocksuite/store';
import JSZip from 'jszip';
import {
blobsFromAssets,
type ImportBatch,
type ImportDoc,
type ImportFolder,
type ImportTag,
type ImportWarning,
} from './import-batch.js';
import { createCollectionDocCRUD } from './markdown.js';
/** Recursive tree node representing a tag-based folder hierarchy. */
type FolderHierarchy = {
export type BearFolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
children: Map<string, BearFolderHierarchy>;
pageId?: string;
parentPath?: string;
};
@@ -29,10 +42,11 @@ type BearImportOptions = {
extensions: ExtensionType[];
};
type BearImportResult = {
export type PlanBearBackupResult = {
docIds: string[];
tags: Map<string, string[]>;
folderHierarchy: FolderHierarchy;
folderHierarchy: BearFolderHierarchy;
batch: ImportBatch;
};
type BundleEntry = {
@@ -182,8 +196,8 @@ function deduplicateTags(tags: string[]): string[] {
*/
function buildFolderHierarchyFromTags(
tagDocMap: Map<string, string[]>
): FolderHierarchy {
const root: FolderHierarchy = {
): BearFolderHierarchy {
const root: BearFolderHierarchy = {
name: '',
path: '',
children: new Map(),
@@ -226,6 +240,34 @@ function buildFolderHierarchyFromTags(
return root;
}
function flattenFolderHierarchy(root: BearFolderHierarchy): ImportFolder[] {
const folders: ImportFolder[] = [];
const visit = (node: BearFolderHierarchy) => {
if (node.name) {
folders.push({
path: node.path,
name: node.name,
parentPath: node.parentPath,
pageId: node.pageId,
});
}
for (const child of node.children.values()) {
visit(child);
}
};
for (const child of root.children.values()) {
visit(child);
}
return folders;
}
function tagsToBatchTags(tags: Map<string, string[]>): ImportTag[] {
return Array.from(tags, ([name, docIds]) => ({ name, docIds }));
}
const GFM_CALLOUT_MAP: Record<string, string> = {
IMPORTANT: '\u26A0',
NOTE: '\uD83D\uDCDD',
@@ -250,9 +292,9 @@ function convertGfmCallouts(markdown: string): string {
if (!inCodeBlock) {
lines[i] = lines[i].replace(
/^(>\s*)\[!(\w+)\]/,
(_match, prefix: string, type: string) => {
(match, prefix: string, type: string) => {
const emoji = GFM_CALLOUT_MAP[type.toUpperCase()];
return emoji ? `${prefix}[!${emoji}]` : _match;
return emoji ? `${prefix}[!${emoji}]` : match;
}
);
}
@@ -306,7 +348,7 @@ function convertHighlights(markdown: string): string {
if (!inCodeBlock) {
lines[i] = lines[i].replace(
/==(\S(?:[^=]|=[^=])*?)==/g,
(_match, content: string) => {
(...[, content]) => {
const firstChar = String.fromCodePoint(content.codePointAt(0)!);
const color = HIGHLIGHT_COLOR_MAP[firstChar];
if (color) {
@@ -340,16 +382,24 @@ function extractTitle(markdown: string, bundleName: string): string {
return bundleName.replace(/\.textbundle$/i, '') || 'Untitled';
}
function applySnapshotTitle(snapshot: DocSnapshot, title: string) {
snapshot.meta.title = title;
snapshot.blocks.props.title = {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
};
}
/**
* Import a Bear .bear2bk backup file.
* Uses JSZip for lazy/streaming decompression to handle large backups.
*/
async function importBearBackup({
async function planBearBackup({
collection,
schema,
imported,
extensions,
}: BearImportOptions): Promise<BearImportResult> {
}: BearImportOptions): Promise<PlanBearBackupResult> {
const provider = getProvider(extensions);
// JSZip reads the zip directory without decompressing all entries
@@ -358,7 +408,7 @@ async function importBearBackup({
// Scan entries and group by textbundle
const bundleMap = new Map<string, BundleEntry>();
zip.forEach((path, _entry) => {
zip.forEach(path => {
if (path.includes('__MACOSX') || path.includes('.DS_Store')) return;
const tbMatch = path.match(/^(.+?\.textbundle)\/(.*)/i);
@@ -421,6 +471,12 @@ async function importBearBackup({
}
const docIds: string[] = [];
const docs: ImportDoc[] = [];
const importBlobs = new Map<
string,
Awaited<ReturnType<typeof blobsFromAssets>>[0]
>();
const warnings: ImportWarning[] = [];
const tagDocMap = new Map<string, string[]>();
// Process bundles sequentially to limit memory.
@@ -497,44 +553,70 @@ async function importBearBackup({
}
const mdAdapter = new MarkdownAdapter(job, provider);
const doc = await mdAdapter.toDoc({
const snapshot = await mdAdapter.toDocSnapshot({
file: markdown,
assets: job.assetsManager,
});
if (doc) {
docIds.push(doc.id);
const metaPatch: Record<string, unknown> = {};
if (bearMeta?.creationDate) {
const ts = Date.parse(String(bearMeta.creationDate));
if (!isNaN(ts)) metaPatch.createDate = ts;
}
if (bearMeta?.modificationDate) {
const ts = Date.parse(String(bearMeta.modificationDate));
if (!isNaN(ts)) metaPatch.updatedDate = ts;
}
if (Object.keys(metaPatch).length) {
collection.meta.setDocMeta(doc.id, metaPatch);
}
for (const tag of tags) {
if (!tagDocMap.has(tag)) {
tagDocMap.set(tag, []);
}
tagDocMap.get(tag)!.push(doc.id);
}
const docId = collection.idGenerator();
snapshot.meta.id = docId;
applySnapshotTitle(snapshot, title);
const meta: ImportDoc['meta'] = { title };
if (bearMeta?.creationDate) {
const ts = Date.parse(String(bearMeta.creationDate));
if (!isNaN(ts)) meta.createDate = ts;
}
} catch (err) {
console.warn(`Failed to import bundle: ${entry.bundlePath}`, err);
if (bearMeta?.modificationDate) {
const ts = Date.parse(String(bearMeta.modificationDate));
if (!isNaN(ts)) meta.updatedDate = ts;
}
docs.push({
id: docId,
snapshot,
meta,
});
docIds.push(docId);
for (const tag of tags) {
if (!tagDocMap.has(tag)) {
tagDocMap.set(tag, []);
}
tagDocMap.get(tag)!.push(docId);
}
for (const blob of await blobsFromAssets(
pendingAssets,
pendingPathBlobIdMap
)) {
importBlobs.set(blob.blobId, blob);
}
} catch {
warnings.push({
code: 'bear-bundle-import-failed',
message: `Failed to import bundle: ${entry.bundlePath}`,
sourcePath: entry.bundlePath,
});
}
}
const folderHierarchy = buildFolderHierarchyFromTags(tagDocMap);
return { docIds, tags: tagDocMap, folderHierarchy };
return {
docIds,
tags: tagDocMap,
folderHierarchy,
batch: {
docs,
blobs: Array.from(importBlobs.values()),
folders: flattenFolderHierarchy(folderHierarchy),
tags: tagsToBatchTags(tagDocMap),
warnings,
done: true,
},
};
}
/** Public API for importing Bear .bear2bk backup archives. */
export const BearTransformer = {
importBearBackup,
planBearBackup,
};
@@ -0,0 +1,157 @@
import {
type DocMeta,
type DocSnapshot,
type Schema,
Transformer,
type Workspace,
} from '@blocksuite/store';
export type ImportBlob = {
blobId: string;
sourcePath: string;
fileName: string;
mime: string;
bytes: Uint8Array;
};
export type ImportIconData =
| {
type: 'emoji';
unicode: string;
}
| {
type: 'image';
content: string;
};
export type ImportFolder = {
path: string;
name: string;
parentPath?: string;
pageId?: string;
icon?: ImportIconData;
};
export type ImportDoc = {
id: string;
sourcePath?: string;
snapshot: DocSnapshot;
meta?: Partial<
Pick<
DocMeta,
'title' | 'createDate' | 'updatedDate' | 'tags' | 'favorite' | 'trash'
>
>;
};
export type ImportTag = {
name: string;
docIds: string[];
};
export type ImportIcon = {
docId: string;
icon: ImportIconData;
};
export type ImportWarning = {
code: string;
message: string;
sourcePath?: string;
};
export type ImportBatch = {
docs: ImportDoc[];
blobs: ImportBlob[];
folders?: ImportFolder[];
tags?: ImportTag[];
icons?: ImportIcon[];
warnings?: ImportWarning[];
progress?: {
completed: number;
total: number;
};
entryId?: string;
isWorkspaceFile?: boolean;
done: boolean;
};
export type ImportCommitResult = {
docIds: string[];
entryId?: string;
isWorkspaceFile?: boolean;
rootFolderId?: string;
warnings: ImportWarning[];
};
export async function blobsFromAssets(
assets: ReadonlyMap<string, File>,
pathBlobIdMap: ReadonlyMap<string, string> = new Map()
) {
const sourcePathByBlobId = new Map<string, string>();
for (const [path, blobId] of pathBlobIdMap) {
sourcePathByBlobId.set(blobId, path);
}
return Promise.all(
Array.from(
assets,
async ([blobId, file]): Promise<ImportBlob> => ({
blobId,
sourcePath: sourcePathByBlobId.get(blobId) ?? file.name,
fileName: file.name,
mime: file.type,
bytes: new Uint8Array(await file.arrayBuffer()),
})
)
);
}
function copyToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
export async function commitImportBatchToWorkspace(
collection: Workspace,
schema: Schema,
batch: ImportBatch
): Promise<ImportCommitResult> {
for (const blob of batch.blobs) {
await collection.blobSync.set(
blob.blobId,
new File([copyToArrayBuffer(blob.bytes)], blob.fileName, {
type: blob.mime,
})
);
}
const transformer = new Transformer({
schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc(id).getStore({ id }),
get: (id: string) => collection.getDoc(id)?.getStore({ id }) ?? null,
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [],
});
const docIds: string[] = [];
for (const doc of batch.docs) {
const store = await transformer.snapshotToDoc(doc.snapshot);
if (!store) continue;
docIds.push(store.id);
if (doc.meta && Object.keys(doc.meta).length) {
collection.meta.setDocMeta(store.id, doc.meta);
}
}
return {
docIds,
entryId: batch.entryId,
isWorkspaceFile: batch.isWorkspaceFile,
warnings: batch.warnings ?? [],
};
}
@@ -1,9 +1,22 @@
export { BearTransformer } from './bear.js';
export { DocxTransformer } from './docx.js';
export { HtmlTransformer } from './html.js';
export {
blobsFromAssets,
commitImportBatchToWorkspace,
type ImportBatch,
type ImportBlob,
type ImportCommitResult,
type ImportDoc,
type ImportFolder,
type ImportIcon,
type ImportIconData,
type ImportTag,
type ImportWarning,
} from './import-batch.js';
export { MarkdownTransformer } from './markdown.js';
export { NotionHtmlTransformer } from './notion-html.js';
export { ObsidianTransformer } from './obsidian.js';
export { PdfTransformer } from './pdf.js';
export { createAssetsArchive, download } from './utils.js';
export { createAssetsArchive, download, Unzip } from './utils.js';
export { ZipTransformer } from './zip.js';
@@ -16,6 +16,7 @@ import { BlockSuiteError, ErrorCode } from '@blocksuite/global/exceptions';
import { sha } from '@blocksuite/global/utils';
import type {
DocMeta,
DocSnapshot,
ExtensionType,
Schema,
Store,
@@ -23,6 +24,12 @@ import type {
} from '@blocksuite/store';
import { extMimeMap, Transformer } from '@blocksuite/store';
import {
blobsFromAssets,
type ImportBatch,
type ImportDoc,
type ImportFolder,
} from './import-batch.js';
import type { AssetMap, ImportedFileEntry, PathBlobIdMap } from './type.js';
import { createAssetsArchive, download, parseMatter, Unzip } from './utils.js';
@@ -438,6 +445,14 @@ function prepareNotionMarkdownFile({
};
}
function applySnapshotTitle(snapshot: DocSnapshot, title: string) {
snapshot.meta.title = title;
snapshot.blocks.props.title = {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
};
}
/**
* Filters hidden/system entries that should never participate in imports.
*/
@@ -685,7 +700,7 @@ async function importMarkdownToDoc({
* @param options.imported The zip file as a Blob
* @returns A Promise that resolves to an array of IDs of the newly created docs
*/
type FolderHierarchy = {
export type FolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
@@ -693,18 +708,19 @@ type FolderHierarchy = {
parentPath?: string;
};
export type ImportMarkdownZipResult = {
export type PlanMarkdownZipResult = {
docIds: string[];
folderHierarchy?: FolderHierarchy;
batch: ImportBatch;
};
async function importMarkdownZip({
async function planMarkdownZip({
collection,
schema,
imported,
extensions,
}: ImportMarkdownZipOptions): Promise<ImportMarkdownZipResult> {
return importMarkdownZipInternal({
}: ImportMarkdownZipOptions): Promise<PlanMarkdownZipResult> {
return planMarkdownZipInternal({
collection,
schema,
imported,
@@ -712,13 +728,13 @@ async function importMarkdownZip({
});
}
async function importNotionMarkdownZip({
async function planNotionMarkdownZip({
collection,
schema,
imported,
extensions,
}: ImportMarkdownZipOptions): Promise<ImportMarkdownZipResult> {
return importMarkdownZipInternal({
}: ImportMarkdownZipOptions): Promise<PlanMarkdownZipResult> {
return planMarkdownZipInternal({
collection,
schema,
imported,
@@ -731,7 +747,7 @@ async function importNotionMarkdownZip({
});
}
async function importMarkdownZipInternal({
async function planMarkdownZipInternal({
collection,
schema,
imported,
@@ -741,12 +757,13 @@ async function importMarkdownZipInternal({
prepareMarkdownFile = prepareDefaultMarkdownFile,
preserveCommonRoot = false,
recursiveZip = false,
}: ImportMarkdownZipInternalOptions): Promise<ImportMarkdownZipResult> {
}: ImportMarkdownZipInternalOptions): Promise<PlanMarkdownZipResult> {
const provider = getProvider([
markdownZipDocLinkToDeltaMatcher,
...extensions,
]);
const docIds: string[] = [];
const docs: ImportDoc[] = [];
const pendingAssets: AssetMap = new Map();
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
const docPathMap: Array<{ fullPath: string; docId: string }> = [];
@@ -812,16 +829,20 @@ async function importMarkdownZipInternal({
assets: job.assetsManager,
});
snapshot.meta.id = pageId;
const doc = await job.snapshotToDoc(snapshot);
if (doc) {
applyMetaPatch(collection, doc.id, meta);
docIds.push(doc.id);
docPathMap.push({ fullPath, docId: doc.id });
}
applySnapshotTitle(snapshot, preferredTitle);
docs.push({
id: pageId,
snapshot,
meta: {
...meta,
title: preferredTitle,
},
});
docIds.push(pageId);
docPathMap.push({ fullPath, docId: pageId });
})
);
// Build folder hierarchy from zip paths
const folderHierarchy = buildMarkdownZipFolderHierarchy(
docPathMap,
normalizeFolderName,
@@ -829,7 +850,18 @@ async function importMarkdownZipInternal({
createRootFolderForTopLevelDocs
);
return { docIds, folderHierarchy };
return {
docIds,
folderHierarchy,
batch: {
docs,
blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap),
folders: folderHierarchy
? flattenFolderHierarchy(folderHierarchy)
: undefined,
done: true,
},
};
}
/**
@@ -921,10 +953,34 @@ function buildMarkdownZipFolderHierarchy(
return root.children.size > 0 ? root : undefined;
}
function flattenFolderHierarchy(root: FolderHierarchy): ImportFolder[] {
const folders: ImportFolder[] = [];
const visit = (node: FolderHierarchy) => {
if (node.name) {
folders.push({
path: node.path,
name: node.name,
parentPath: node.parentPath,
pageId: node.pageId,
});
}
for (const child of node.children.values()) {
visit(child);
}
};
for (const child of root.children.values()) {
visit(child);
}
return folders;
}
export const MarkdownTransformer = {
exportDoc,
importMarkdownToBlock,
importMarkdownToDoc,
importMarkdownZip,
importNotionMarkdownZip,
planMarkdownZip,
planNotionMarkdownZip,
};
@@ -12,6 +12,14 @@ import {
type Workspace,
} from '@blocksuite/store';
import {
blobsFromAssets,
type ImportBatch,
type ImportDoc,
type ImportFolder,
type ImportIconData,
type ImportWarning,
} from './import-batch.js';
import { Unzip } from './utils.js';
type ImportNotionZipOptions = {
@@ -21,12 +29,12 @@ type ImportNotionZipOptions = {
extensions: ExtensionType[];
};
type PageIcon = {
export type PageIcon = {
type: 'emoji' | 'image';
content: string; // emoji unicode or image URL/data
content: string;
};
type FolderHierarchy = {
export type FolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
@@ -35,12 +43,13 @@ type FolderHierarchy = {
icon?: PageIcon;
};
type ImportNotionZipResult = {
export type PlanNotionHtmlZipResult = {
entryId: string | undefined;
pageIds: string[];
isWorkspaceFile: boolean;
hasMarkdown: boolean;
folderHierarchy?: FolderHierarchy;
batch: ImportBatch;
};
function getProvider(extensions: ExtensionType[]) {
@@ -61,22 +70,9 @@ function parseFolderPath(filePath: string): {
}
function extractPageIcon(doc: Document): PageIcon | undefined {
// Look for Notion page icon in the HTML
// Notion export format: <div class="page-header-icon undefined"><span class="icon">✅</span></div>
console.log('=== Extracting page icon ===');
// Check if there's a head section with title for debugging
const headTitle = doc.querySelector('head title');
if (headTitle) {
console.log('Page title from head:', headTitle.textContent);
}
// Look for the exact Notion export structure: .page-header-icon .icon
const notionIconSpan = doc.querySelector('.page-header-icon .icon');
if (notionIconSpan && notionIconSpan.textContent) {
const iconContent = notionIconSpan.textContent.trim();
console.log('Found Notion icon (.page-header-icon .icon):', iconContent);
if (/\p{Emoji}/u.test(iconContent)) {
return {
type: 'emoji',
@@ -85,50 +81,29 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
// Look for page header area for debugging
const pageHeader = doc.querySelector('.page-header-icon');
if (pageHeader) {
console.log(
'Found .page-header-icon:',
pageHeader.outerHTML.substring(0, 300) + '...'
);
}
// Fallback: try to find emoji icons with older selectors
const emojiIcon = doc.querySelector('.page-header-icon .notion-emoji');
if (emojiIcon && emojiIcon.textContent) {
console.log(
'Found emoji icon (.page-header-icon .notion-emoji):',
emojiIcon.textContent
);
return {
type: 'emoji',
content: emojiIcon.textContent.trim(),
};
}
// Try alternative emoji selectors
const altEmojiIcon = doc.querySelector('[role="img"][aria-label]');
if (
altEmojiIcon &&
altEmojiIcon.textContent &&
/\p{Emoji}/u.test(altEmojiIcon.textContent)
) {
console.log(
'Found emoji icon ([role="img"][aria-label]):',
altEmojiIcon.textContent
);
return {
type: 'emoji',
content: altEmojiIcon.textContent.trim(),
};
}
// Look for image icons in the page header
const imageIcon = doc.querySelector('.page-header-icon img');
if (imageIcon) {
const src = imageIcon.getAttribute('src');
console.log('Found image icon (.page-header-icon img):', src);
if (src) {
return {
type: 'image',
@@ -137,27 +112,15 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
// Fallback: Look for any span with emoji class "icon" in page header area
const iconSpans = doc.querySelectorAll('span.icon');
for (const span of iconSpans) {
if (span.textContent && /\p{Emoji}/u.test(span.textContent.trim())) {
const parent = span.parentElement;
console.log(
'Found emoji in span.icon:',
span.textContent,
'parent classes:',
parent?.className
);
// Check if this is in a page header context
if (
parent &&
(parent.classList.contains('page-header-icon') ||
parent.closest('.page-header-icon'))
) {
console.log(
'Using emoji from span.icon in page header:',
span.textContent
);
return {
type: 'emoji',
content: span.textContent.trim(),
@@ -166,15 +129,11 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
// Fallback: Try to find icons in the page title area that might contain emoji
const pageTitle = doc.querySelector('.page-title, h1');
if (pageTitle && pageTitle.textContent) {
console.log('Page title element found:', pageTitle.textContent);
const text = pageTitle.textContent.trim();
// Check if the title starts with an emoji
const emojiMatch = text.match(/^(\p{Emoji}+)/u);
if (emojiMatch) {
console.log('Found emoji in title:', emojiMatch[1]);
return {
type: 'emoji',
content: emojiMatch[1],
@@ -182,7 +141,6 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
console.log('No page icon found');
return undefined;
}
@@ -200,7 +158,6 @@ function buildFolderHierarchy(
let current = root;
let currentPath = '';
// Navigate/create folder structure
for (const folderName of folderParts) {
const parentPath = currentPath;
currentPath = currentPath ? `${currentPath}/${folderName}` : folderName;
@@ -216,7 +173,6 @@ function buildFolderHierarchy(
current = current.children.get(folderName)!;
}
// If this is a page file, associate it with the current folder
if (fileName.endsWith('.html') && !fileName.startsWith('index.html')) {
const pageName = fileName.replace(/\.html$/, '');
if (!current.children.has(pageName)) {
@@ -229,7 +185,6 @@ function buildFolderHierarchy(
icon: icon,
});
} else {
// Update existing entry with pageId and icon
const existingPage = current.children.get(pageName)!;
existingPage.pageId = pageId;
if (icon) {
@@ -242,29 +197,59 @@ function buildFolderHierarchy(
return root;
}
/**
* Imports a Notion zip file into the BlockSuite collection.
*
* @param options - The options for importing.
* @param options.collection - The BlockSuite document collection.
* @param options.schema - The schema of the BlockSuite document collection.
* @param options.imported - The imported zip file as a Blob.
*
* @returns A promise that resolves to an object containing:
* - entryId: The ID of the entry page (if any).
* - pageIds: An array of imported page IDs.
* - isWorkspaceFile: Whether the imported file is a workspace file.
* - hasMarkdown: Whether the zip contains markdown files.
* - folderHierarchy: The parsed folder hierarchy from the Notion export.
*/
async function importNotionZip({
function toImportIconData(icon?: PageIcon): ImportIconData | undefined {
if (!icon) return undefined;
if (icon.type === 'emoji') {
return {
type: 'emoji',
unicode: icon.content,
};
}
return {
type: 'image',
content: icon.content,
};
}
function flattenFolderHierarchy(root: FolderHierarchy): ImportFolder[] {
const folders: ImportFolder[] = [];
const visit = (node: FolderHierarchy) => {
if (node.name) {
folders.push({
path: node.path,
name: node.name,
parentPath: node.parentPath,
pageId: node.pageId,
icon: toImportIconData(node.icon),
});
}
for (const child of node.children.values()) {
visit(child);
}
};
for (const child of root.children.values()) {
visit(child);
}
return folders;
}
async function planNotionHtmlZip({
collection,
schema,
imported,
extensions,
}: ImportNotionZipOptions): Promise<ImportNotionZipResult> {
}: ImportNotionZipOptions): Promise<PlanNotionHtmlZipResult> {
const provider = getProvider(extensions);
const pageIds: string[] = [];
const docs: ImportDoc[] = [];
const blobs = new Map<
string,
Awaited<ReturnType<typeof blobsFromAssets>>[0]
>();
const warnings: ImportWarning[] = [];
let isWorkspaceFile = false;
let hasMarkdown = false;
let entryId: string | undefined;
@@ -280,7 +265,7 @@ async function importNotionZip({
const pageMap = new Map<string, string>();
const pagePaths: string[] = [];
const promises: Promise<void>[] = [];
const pendingAssets = new Map<string, Blob>();
const pendingAssets = new Map<string, File>();
const pendingPathBlobIdMap = new Map<string, string>();
for (const { path, content, index } of unzip) {
if (path.startsWith('__MACOSX/')) continue;
@@ -305,11 +290,7 @@ async function importNotionZip({
const text = await content.text();
const doc = new DOMParser().parseFromString(text, 'text/html');
const pageBody = doc.querySelector('.page-body');
if (pageBody && pageBody.children.length === 0) {
// Skip empty pages
continue;
}
// Extract page icon from the HTML
if (pageBody && pageBody.children.length === 0) continue;
pageIcon = extractPageIcon(doc);
}
@@ -327,10 +308,12 @@ async function importNotionZip({
continue;
}
if (index === 0 && fileName.endsWith('.csv')) {
window.open(
'https://affine.pro/blog/import-your-data-from-notion-into-affine',
'_blank'
);
warnings.push({
code: 'notion-csv-export',
message:
'The imported Notion export appears to be CSV instead of HTML.',
sourcePath: path,
});
continue;
}
if (fileName.endsWith('.zip')) {
@@ -342,7 +325,7 @@ async function importNotionZip({
}
const blob = content;
const ext = path.split('.').at(-1) ?? '';
const mime = extMimeMap.get(ext) ?? '';
const mime = extMimeMap.get(ext.toLowerCase()) ?? '';
const key = await sha(await blob.arrayBuffer());
const filePathSplit = path.split('/');
while (filePathSplit.length > 1) {
@@ -375,32 +358,57 @@ async function importNotionZip({
pathBlobIdMap.set(key, value);
}
}
const page = await htmlAdapter.toDoc({
const snapshot = await htmlAdapter.toDocSnapshot({
file: await zipFile.get(path)!.text(),
pageId: pageMap.get(path),
pageMap,
assets: job.assetsManager,
});
if (page) {
pageIds.push(page.id);
}
docs.push({
id: snapshot.meta.id,
snapshot,
});
pageIds.push(snapshot.meta.id);
});
promises.push(...pagePromises);
promises.push(
blobsFromAssets(pendingAssets, pendingPathBlobIdMap).then(importBlobs => {
for (const blob of importBlobs) {
blobs.set(blob.blobId, blob);
}
})
);
return promises;
};
const allPromises = await parseZipFile(imported);
await Promise.all(allPromises.flat());
entryId = entryId ?? pageIds[0];
// Build folder hierarchy from collected paths
const folderHierarchy =
pagePathsWithIds.length > 0
? buildFolderHierarchy(pagePathsWithIds)
: undefined;
return { entryId, pageIds, isWorkspaceFile, hasMarkdown, folderHierarchy };
return {
entryId,
pageIds,
isWorkspaceFile,
hasMarkdown,
folderHierarchy,
batch: {
docs,
blobs: Array.from(blobs.values()),
folders: folderHierarchy
? flattenFolderHierarchy(folderHierarchy)
: undefined,
warnings,
entryId,
isWorkspaceFile,
done: true,
},
};
}
export const NotionHtmlTransformer = {
importNotionZip,
planNotionHtmlZip,
};
@@ -20,7 +20,11 @@ import { extMimeMap, nanoid } from '@blocksuite/store';
import type { Html, Text } from 'mdast';
import {
applyMetaPatch,
blobsFromAssets,
type ImportBatch,
type ImportDoc,
} from './import-batch.js';
import {
bindImportedAssetsToJob,
createMarkdownImportJob,
getProvider,
@@ -129,8 +133,8 @@ function preprocessTitleHeader(markdown: string): string {
function preprocessObsidianCallouts(markdown: string): string {
return markdown.replace(
/^(> *)\[!([^\]\n]+)\]([+-]?)([^\n]*)/gm,
(_, prefix, type, _fold, rest) => {
/^(> *)\[!([^\]\n]+)\](?:[+-]?)([^\n]*)/gm,
(_, prefix, type, rest) => {
const calloutToken =
CALLOUT_TYPE_MAP[type.trim().toLowerCase()] ?? DEFAULT_CALLOUT_EMOJI;
const title = rest.trim();
@@ -688,12 +692,16 @@ export type ImportObsidianVaultResult = {
docEmojis: Map<string, string>;
};
export async function importObsidianVault({
export type PlanObsidianVaultResult = ImportObsidianVaultResult & {
batch: ImportBatch;
};
export async function planObsidianVault({
collection,
schema,
importedFiles,
extensions,
}: ImportObsidianVaultOptions): Promise<ImportObsidianVaultResult> {
}: ImportObsidianVaultOptions): Promise<PlanObsidianVaultResult> {
const provider = getProvider([
obsidianWikilinkToDeltaMatcher,
obsidianAttachmentEmbedMarkdownAdapterMatcher,
@@ -701,6 +709,7 @@ export async function importObsidianVault({
]);
const docIds: string[] = [];
const docs: ImportDoc[] = [];
const docEmojis = new Map<string, string>();
const pendingAssets: AssetMap = new Map();
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
@@ -813,22 +822,31 @@ export async function importObsidianVault({
if (snapshot) {
snapshot.meta.id = predefinedId;
const doc = await job.snapshotToDoc(snapshot);
if (doc) {
applyMetaPatch(collection, doc.id, {
...meta,
title: preferredTitle,
trash: false,
});
docIds.push(doc.id);
}
docs.push({
id: predefinedId,
snapshot,
meta: { ...meta, title: preferredTitle, trash: false },
});
docIds.push(predefinedId);
}
})
);
return { docIds, docEmojis };
return {
docIds,
docEmojis,
batch: {
docs,
blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap),
icons: Array.from(docEmojis, ([docId, emoji]) => ({
docId,
icon: { type: 'emoji', unicode: emoji },
})),
done: true,
},
};
}
export const ObsidianTransformer = {
importObsidianVault,
planObsidianVault,
};