mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-08-31 05:39:28 +08:00
feat(core): support Notion markdown zip imports (#14910)
## What changed - Markdown zip imports now resolve local `.md` links like `./test/2.md` into AFFiNE linked-page references when the target document exists in the same archive. - Added Notion Markdown `.zip` import support in the desktop import dialog, including nested zip traversal, Notion title extraction, hash-stripped folder names, attachments, and folder hierarchy integration. - Added i18n entries and adapter coverage for standard Markdown zip links and Notion Markdown zip imports. ## Why Markdown and Notion exports often contain links between notes using relative `.md` paths. Keeping those as plain URLs makes imported workspaces harder to navigate, so the importer now preassigns document ids and rewrites resolvable archive-local markdown links into linked pages. ## Notes The markdown zip folder hierarchy implementation now comes from latest `origin/canary`, so this PR only layers relative-link resolution and Notion Markdown zip support on top of that upstream behavior. ## Validation - `yarn vitest --run blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts` - `yarn tsc -b blocksuite/affine/all/tsconfig.json --verbose` - `git diff --check` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added support for importing Notion Markdown exports in `.zip` format, including subpages, attachments, and nested folders. * Internal links inside imported Markdown now resolve correctly between pages, preserving link text when available. * The import dialog now includes a dedicated “Notion (Markdown, .zip)” option. * **Bug Fixes** * Improved filename handling so non-Latin characters in ZIP imports are preserved correctly. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: DarkSky <25152247+darkskygit@users.noreply.github.com> Co-authored-by: DarkSky <darksky2048@gmail.com>
This commit is contained in:
@@ -30,6 +30,7 @@ import type {
|
||||
} from '@blocksuite/store';
|
||||
import { AssetsManager, MemoryBlobCRUD, Schema } from '@blocksuite/store';
|
||||
import { TestWorkspace } from '@blocksuite/store/test';
|
||||
import * as fflate from 'fflate';
|
||||
import { describe, expect, test } from 'vitest';
|
||||
|
||||
import { AffineSchemas } from '../../schemas.js';
|
||||
@@ -64,6 +65,25 @@ function markdownFixture(relativePath: string): File {
|
||||
);
|
||||
}
|
||||
|
||||
function zipBytes(entries: Record<string, string | Uint8Array>) {
|
||||
return fflate.zipSync(
|
||||
Object.fromEntries(
|
||||
Object.entries(entries).map(([path, content]) => [
|
||||
path,
|
||||
typeof content === 'string' ? fflate.strToU8(content) : content,
|
||||
])
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function zipFixture(entries: Record<string, string | Uint8Array>) {
|
||||
const zipped = zipBytes(entries);
|
||||
const buffer = new ArrayBuffer(zipped.byteLength);
|
||||
new Uint8Array(buffer).set(zipped);
|
||||
|
||||
return new Blob([buffer], { type: 'application/zip' });
|
||||
}
|
||||
|
||||
function exportSnapshot(doc: Store): DocSnapshot {
|
||||
const job = doc.getTransformer([
|
||||
docLinkBaseURLMiddleware(doc.workspace.id),
|
||||
@@ -74,6 +94,17 @@ function exportSnapshot(doc: Store): DocSnapshot {
|
||||
return snapshot!;
|
||||
}
|
||||
|
||||
function noteSnapshotByTitle(collection: TestWorkspace, title: string) {
|
||||
const meta = collection.meta.docMetas.find(meta => meta.title === title);
|
||||
expect(meta).toBeTruthy();
|
||||
const doc = collection.getDoc(meta!.id)?.getStore({ id: meta!.id });
|
||||
expect(doc).toBeTruthy();
|
||||
const snapshot = exportSnapshot(doc!);
|
||||
return snapshot.blocks.children.find(
|
||||
block => block.flavour === 'affine:note'
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDeltaForSnapshot(
|
||||
delta: DeltaInsert<AffineTextAttributes>[],
|
||||
titleById: ReadonlyMap<string, string>
|
||||
@@ -172,6 +203,21 @@ function snapshotDocByTitle(
|
||||
return simplifyBlockForSnapshot(exportSnapshot(doc!).blocks, titleById);
|
||||
}
|
||||
|
||||
function collectSimplifiedDeltas(
|
||||
block: Record<string, unknown>
|
||||
): Record<string, unknown>[] {
|
||||
const deltas = Array.isArray(block.delta)
|
||||
? (block.delta as Record<string, unknown>[])
|
||||
: [];
|
||||
const childDeltas = Array.isArray(block.children)
|
||||
? (block.children as Record<string, unknown>[]).flatMap(child =>
|
||||
collectSimplifiedDeltas(child)
|
||||
)
|
||||
: [];
|
||||
|
||||
return [...deltas, ...childDeltas];
|
||||
}
|
||||
|
||||
describe('snapshot to markdown', () => {
|
||||
test('code', async () => {
|
||||
const blockSnapshot: BlockSnapshot = {
|
||||
@@ -318,6 +364,275 @@ Hello world
|
||||
expect(exported.file).toContain('> \\- Oranges');
|
||||
});
|
||||
|
||||
test('imports notion markdown zip titles and folder names', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
collection.storeExtensions = testStoreExtensions;
|
||||
collection.meta.initialize();
|
||||
|
||||
const imported = zipFixture({
|
||||
'Notion Export/Workspace 11111111111111111111111111111111.md':
|
||||
'# Workspace\nRoot body',
|
||||
'Notion Export/Workspace 11111111111111111111111111111111/Nested Page 22222222222222222222222222222222.md':
|
||||
'# Nested Page\nNested body',
|
||||
});
|
||||
|
||||
const { docIds, folderHierarchy } =
|
||||
await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions: testStoreExtensions,
|
||||
});
|
||||
|
||||
expect(docIds).toHaveLength(2);
|
||||
expect(
|
||||
collection.meta.docMetas
|
||||
.map(meta => meta.title)
|
||||
.sort((a, b) => (a ?? '').localeCompare(b ?? ''))
|
||||
).toEqual(['Nested Page', 'Workspace']);
|
||||
|
||||
const nestedNote = noteSnapshotByTitle(collection, 'Nested Page');
|
||||
expect(JSON.stringify(nestedNote)).toContain('Nested body');
|
||||
expect(JSON.stringify(nestedNote)).not.toContain('Nested Page');
|
||||
|
||||
const [folder] = [...(folderHierarchy?.children.values() ?? [])];
|
||||
expect(folder?.name).toBe('Notion Export');
|
||||
const workspaceMeta = collection.meta.docMetas.find(
|
||||
meta => meta.title === 'Workspace'
|
||||
);
|
||||
expect([...folder!.children.values()]).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ pageId: workspaceMeta?.id }),
|
||||
])
|
||||
);
|
||||
const workspaceFolder = [...folder!.children.values()].find(
|
||||
child => child.name === 'Workspace'
|
||||
);
|
||||
const nestedMeta = collection.meta.docMetas.find(
|
||||
meta => meta.title === 'Nested Page'
|
||||
);
|
||||
expect([...workspaceFolder!.children.values()]).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ pageId: nestedMeta?.id }),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
test('imports notion markdown zip folders with CJK names', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
collection.storeExtensions = testStoreExtensions;
|
||||
collection.meta.initialize();
|
||||
|
||||
const imported = zipFixture({
|
||||
'Export/工作 11111111111111111111111111111111.md': '# 工作\nRoot body',
|
||||
'Export/工作 11111111111111111111111111111111/SDK架构 22222222222222222222222222222222.md':
|
||||
'# SDK架构\nNested body',
|
||||
});
|
||||
|
||||
const { folderHierarchy } =
|
||||
await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions: testStoreExtensions,
|
||||
});
|
||||
|
||||
const [rootFolder] = [...(folderHierarchy?.children.values() ?? [])];
|
||||
expect(rootFolder?.name).toBe('Export');
|
||||
const workFolder = [...(rootFolder?.children.values() ?? [])].find(
|
||||
child => child.name === '工作'
|
||||
);
|
||||
expect(workFolder?.name).toBe('工作');
|
||||
expect([...workFolder!.children.values()]).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ pageId: expect.any(String) }),
|
||||
])
|
||||
);
|
||||
});
|
||||
|
||||
test('imports notion markdown zip title from frontmatter when heading is absent', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
collection.storeExtensions = testStoreExtensions;
|
||||
collection.meta.initialize();
|
||||
|
||||
const imported = zipFixture({
|
||||
'Export/Fallback 11111111111111111111111111111111.md':
|
||||
'---\ntitle: Frontmatter Title\n---\nBody',
|
||||
});
|
||||
|
||||
const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions: testStoreExtensions,
|
||||
});
|
||||
|
||||
expect(docIds).toHaveLength(1);
|
||||
expect(collection.meta.getDocMeta(docIds[0])?.title).toBe(
|
||||
'Frontmatter Title'
|
||||
);
|
||||
});
|
||||
|
||||
test('imports markdown zip relative doc links as linked pages', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
collection.storeExtensions = testStoreExtensions;
|
||||
collection.meta.initialize();
|
||||
|
||||
const imported = zipFixture({
|
||||
'entry.md': [
|
||||
'[引用](./test/2.md)',
|
||||
'[missing](./missing.md)',
|
||||
'[external](https://example.com/test.md)',
|
||||
].join('\n\n'),
|
||||
'test/2.md': 'target page',
|
||||
});
|
||||
|
||||
const { docIds } = await MarkdownTransformer.importMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions: testStoreExtensions,
|
||||
});
|
||||
expect(docIds).toHaveLength(2);
|
||||
|
||||
const titleById = new Map(
|
||||
collection.meta.docMetas.map(meta => [
|
||||
meta.id,
|
||||
meta.title ?? '<untitled>',
|
||||
])
|
||||
);
|
||||
const entryDeltas = collectSimplifiedDeltas(
|
||||
snapshotDocByTitle(collection, 'entry', titleById)
|
||||
);
|
||||
|
||||
expect(entryDeltas).toContainEqual({
|
||||
insert: ' ',
|
||||
reference: {
|
||||
type: 'LinkedPage',
|
||||
page: '2',
|
||||
title: '引用',
|
||||
},
|
||||
});
|
||||
expect(entryDeltas).toContainEqual({
|
||||
insert: 'missing',
|
||||
link: './missing.md',
|
||||
});
|
||||
expect(entryDeltas).toContainEqual({
|
||||
insert: 'external',
|
||||
link: 'https://example.com/test.md',
|
||||
});
|
||||
});
|
||||
|
||||
test('imports notion markdown zip relative doc links as linked pages', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
collection.storeExtensions = testStoreExtensions;
|
||||
collection.meta.initialize();
|
||||
|
||||
const imported = zipFixture({
|
||||
'Workspace 11111111111111111111111111111111/Entry 22222222222222222222222222222222.md':
|
||||
'# Entry\n[引用](./test/Target%2033333333333333333333333333333333.md)',
|
||||
'Workspace 11111111111111111111111111111111/test/Target 33333333333333333333333333333333.md':
|
||||
'# Target\ntarget page',
|
||||
});
|
||||
|
||||
const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions: testStoreExtensions,
|
||||
});
|
||||
expect(docIds).toHaveLength(2);
|
||||
|
||||
const titleById = new Map(
|
||||
collection.meta.docMetas.map(meta => [
|
||||
meta.id,
|
||||
meta.title ?? '<untitled>',
|
||||
])
|
||||
);
|
||||
const entryDeltas = collectSimplifiedDeltas(
|
||||
snapshotDocByTitle(collection, 'Entry', titleById)
|
||||
);
|
||||
|
||||
expect(entryDeltas).toContainEqual({
|
||||
insert: ' ',
|
||||
reference: {
|
||||
type: 'LinkedPage',
|
||||
page: 'Target',
|
||||
title: '引用',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('imports nested notion markdown zips with isolated relative links', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
collection.storeExtensions = testStoreExtensions;
|
||||
collection.meta.initialize();
|
||||
|
||||
const imported = zipFixture({
|
||||
'Export/Part A.zip': zipBytes({
|
||||
'Entry 11111111111111111111111111111111.md':
|
||||
'# Entry A\n[go](./Target%2022222222222222222222222222222222.md)',
|
||||
'Target 22222222222222222222222222222222.md': '# Target A\nA body',
|
||||
}),
|
||||
'Export/Part B.zip': zipBytes({
|
||||
'Entry 11111111111111111111111111111111.md':
|
||||
'# Entry B\n[go](./Target%2022222222222222222222222222222222.md)',
|
||||
'Target 22222222222222222222222222222222.md': '# Target B\nB body',
|
||||
}),
|
||||
});
|
||||
|
||||
const { docIds, folderHierarchy } =
|
||||
await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions: testStoreExtensions,
|
||||
});
|
||||
expect(docIds).toHaveLength(4);
|
||||
|
||||
const titleById = new Map(
|
||||
collection.meta.docMetas.map(meta => [
|
||||
meta.id,
|
||||
meta.title ?? '<untitled>',
|
||||
])
|
||||
);
|
||||
const entryADeltas = collectSimplifiedDeltas(
|
||||
snapshotDocByTitle(collection, 'Entry A', titleById)
|
||||
);
|
||||
const entryBDeltas = collectSimplifiedDeltas(
|
||||
snapshotDocByTitle(collection, 'Entry B', titleById)
|
||||
);
|
||||
|
||||
expect(entryADeltas).toContainEqual({
|
||||
insert: ' ',
|
||||
reference: {
|
||||
type: 'LinkedPage',
|
||||
page: 'Target A',
|
||||
title: 'go',
|
||||
},
|
||||
});
|
||||
expect(entryBDeltas).toContainEqual({
|
||||
insert: ' ',
|
||||
reference: {
|
||||
type: 'LinkedPage',
|
||||
page: 'Target B',
|
||||
title: 'go',
|
||||
},
|
||||
});
|
||||
|
||||
const [rootFolder] = [...(folderHierarchy?.children.values() ?? [])];
|
||||
expect(rootFolder?.name).toBe('Export');
|
||||
expect(
|
||||
[...(rootFolder?.children.values() ?? [])].map(node => node.name)
|
||||
).toEqual(expect.arrayContaining(['Part A', 'Part B']));
|
||||
});
|
||||
|
||||
test('imports obsidian vault fixtures', async () => {
|
||||
const schema = new Schema().register(AffineSchemas);
|
||||
const collection = new TestWorkspace();
|
||||
|
||||
@@ -3,7 +3,12 @@ import {
|
||||
docLinkBaseURLMiddleware,
|
||||
fileNameMiddleware,
|
||||
filePathMiddleware,
|
||||
FULL_FILE_PATH_KEY,
|
||||
getImageFullPath,
|
||||
MarkdownAdapter,
|
||||
type MarkdownAST,
|
||||
MarkdownASTToDeltaExtension,
|
||||
normalizeFilePathReference,
|
||||
titleMiddleware,
|
||||
} from '@blocksuite/affine-shared/adapters';
|
||||
import { Container } from '@blocksuite/global/di';
|
||||
@@ -61,6 +66,117 @@ const FRONTMATTER_KEYS = {
|
||||
trash: ['trash', 'trashed', 'deleted', 'archived'],
|
||||
};
|
||||
|
||||
const MARKDOWN_ZIP_PAGE_ID_CONFIG_PREFIX = 'markdown-zip:page-id:';
|
||||
|
||||
function normalizeMarkdownZipLookupPath(path: string) {
|
||||
return normalizeFilePathReference(path).toLowerCase();
|
||||
}
|
||||
|
||||
function stripMarkdownExtension(path: string) {
|
||||
return path.replace(/\.md$/i, '');
|
||||
}
|
||||
|
||||
function splitMarkdownLinkTarget(url: string) {
|
||||
const queryIndex = url.indexOf('?');
|
||||
const hashIndex = url.indexOf('#');
|
||||
const splitIndex = [queryIndex, hashIndex]
|
||||
.filter(index => index >= 0)
|
||||
.sort((a, b) => a - b)[0];
|
||||
|
||||
return splitIndex === undefined ? url : url.slice(0, splitIndex);
|
||||
}
|
||||
|
||||
function isLocalMarkdownDocLink(url: string) {
|
||||
const path = splitMarkdownLinkTarget(url).trim();
|
||||
if (!path || path.startsWith('//') || path.startsWith('#')) {
|
||||
return false;
|
||||
}
|
||||
if (/^[a-z][a-z0-9+.-]*:/i.test(path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const fileName = path.split('/').at(-1) ?? '';
|
||||
return path.toLowerCase().endsWith('.md') || !fileName.includes('.');
|
||||
}
|
||||
|
||||
function markdownAstText(ast: MarkdownAST): string {
|
||||
if ('value' in ast && typeof ast.value === 'string') {
|
||||
return ast.value;
|
||||
}
|
||||
if ('children' in ast && Array.isArray(ast.children)) {
|
||||
return ast.children.map(child => markdownAstText(child)).join('');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function getMarkdownZipPageIdConfigKey(path: string) {
|
||||
return `${MARKDOWN_ZIP_PAGE_ID_CONFIG_PREFIX}${normalizeMarkdownZipLookupPath(
|
||||
path
|
||||
)}`;
|
||||
}
|
||||
|
||||
function getMarkdownZipTargetPageId(
|
||||
configs: Map<string, string>,
|
||||
currentFilePath: string,
|
||||
url: string
|
||||
) {
|
||||
const targetPath = splitMarkdownLinkTarget(url);
|
||||
const fullPath = getImageFullPath(currentFilePath, targetPath);
|
||||
const candidates = [fullPath, stripMarkdownExtension(fullPath)];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const pageId = configs.get(getMarkdownZipPageIdConfigKey(candidate));
|
||||
if (pageId) {
|
||||
return pageId;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const markdownZipDocLinkToDeltaMatcher = MarkdownASTToDeltaExtension({
|
||||
name: 'markdown-zip-doc-link',
|
||||
match: ast =>
|
||||
ast.type === 'link' &&
|
||||
'url' in ast &&
|
||||
typeof ast.url === 'string' &&
|
||||
isLocalMarkdownDocLink(ast.url),
|
||||
toDelta: (ast, context) => {
|
||||
if (!('children' in ast) || !('url' in ast)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const currentFilePath = context.configs.get(FULL_FILE_PATH_KEY);
|
||||
const targetPageId =
|
||||
typeof currentFilePath === 'string'
|
||||
? getMarkdownZipTargetPageId(context.configs, currentFilePath, ast.url)
|
||||
: null;
|
||||
|
||||
if (targetPageId) {
|
||||
const title = markdownAstText(ast).trim();
|
||||
return [
|
||||
{
|
||||
insert: ' ',
|
||||
attributes: {
|
||||
reference: {
|
||||
type: 'LinkedPage',
|
||||
pageId: targetPageId,
|
||||
...(title ? { title } : {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
return ast.children.flatMap(child =>
|
||||
context.toDelta(child).map(delta => {
|
||||
delta.attributes = { ...delta.attributes, link: ast.url };
|
||||
return delta;
|
||||
})
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
const truthyStrings = new Set(['true', 'yes', 'y', '1', 'on']);
|
||||
const falsyStrings = new Set(['false', 'no', 'n', '0', 'off']);
|
||||
|
||||
@@ -234,6 +350,94 @@ type ImportMarkdownZipOptions = {
|
||||
extensions: ExtensionType[];
|
||||
};
|
||||
|
||||
type PrepareMarkdownFileOptions = {
|
||||
filename: string;
|
||||
markdown: string;
|
||||
};
|
||||
|
||||
type PreparedMarkdownFile = {
|
||||
content: string;
|
||||
meta: ParsedFrontmatterMeta;
|
||||
preferredTitle: string;
|
||||
};
|
||||
|
||||
type ImportMarkdownZipInternalOptions = ImportMarkdownZipOptions & {
|
||||
createRootFolderForTopLevelDocs?: boolean;
|
||||
normalizeFolderName?: (folderName: string) => string;
|
||||
prepareMarkdownFile?: (
|
||||
options: PrepareMarkdownFileOptions
|
||||
) => PreparedMarkdownFile;
|
||||
preserveCommonRoot?: boolean;
|
||||
recursiveZip?: boolean;
|
||||
};
|
||||
|
||||
function getFileNameWithoutExtension(filename: string) {
|
||||
return filename.replace(/\.[^/.]+$/, '');
|
||||
}
|
||||
|
||||
function stripNotionHash(name: string) {
|
||||
return name
|
||||
.replace(
|
||||
/\s+[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i,
|
||||
''
|
||||
)
|
||||
.replace(/\s+[0-9a-f]{32}$/i, '');
|
||||
}
|
||||
|
||||
function parseNotionMarkdownTitle(markdown: string):
|
||||
| {
|
||||
title: string;
|
||||
content: string;
|
||||
}
|
||||
| undefined {
|
||||
const match = markdown.match(/^\uFEFF?#(?!#)\s+(.+?)\s*(?:\r?\n|$)/);
|
||||
if (!match) {
|
||||
return;
|
||||
}
|
||||
|
||||
const title = match?.[1]?.trim();
|
||||
if (!title) {
|
||||
return;
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
content: markdown.slice(match[0].length),
|
||||
};
|
||||
}
|
||||
|
||||
function prepareDefaultMarkdownFile({
|
||||
filename,
|
||||
markdown,
|
||||
}: PrepareMarkdownFileOptions): PreparedMarkdownFile {
|
||||
const fileNameWithoutExt = getFileNameWithoutExtension(filename);
|
||||
const { content, meta } = parseFrontmatter(markdown);
|
||||
return {
|
||||
content,
|
||||
meta,
|
||||
preferredTitle: meta.title ?? fileNameWithoutExt,
|
||||
};
|
||||
}
|
||||
|
||||
function prepareNotionMarkdownFile({
|
||||
filename,
|
||||
markdown,
|
||||
}: PrepareMarkdownFileOptions): PreparedMarkdownFile {
|
||||
const notionTitle = parseNotionMarkdownTitle(markdown);
|
||||
const { content, meta } = parseFrontmatter(notionTitle?.content ?? markdown);
|
||||
const fallbackTitle = stripNotionHash(getFileNameWithoutExtension(filename));
|
||||
const preferredTitle = notionTitle?.title ?? meta.title ?? fallbackTitle;
|
||||
|
||||
return {
|
||||
content,
|
||||
meta: {
|
||||
...meta,
|
||||
title: preferredTitle,
|
||||
},
|
||||
preferredTitle,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters hidden/system entries that should never participate in imports.
|
||||
*/
|
||||
@@ -331,6 +535,25 @@ export function bindImportedAssetsToJob(
|
||||
return pathBlobIdMap;
|
||||
}
|
||||
|
||||
function bindImportedMarkdownPagesToJob(
|
||||
job: Transformer,
|
||||
pagePathIdMap: ReadonlyMap<string, string>
|
||||
) {
|
||||
for (const [path, pageId] of pagePathIdMap.entries()) {
|
||||
job.adapterConfigs.set(getMarkdownZipPageIdConfigKey(path), pageId);
|
||||
}
|
||||
}
|
||||
|
||||
function registerMarkdownZipPagePath(
|
||||
pagePathIdMap: Map<string, string>,
|
||||
path: string,
|
||||
pageId: string
|
||||
) {
|
||||
const normalizedPath = normalizeFilePathReference(path);
|
||||
pagePathIdMap.set(normalizedPath, pageId);
|
||||
pagePathIdMap.set(stripMarkdownExtension(normalizedPath), pageId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Exports a doc to a Markdown file or a zip archive containing Markdown and assets.
|
||||
* @param doc The doc to export
|
||||
@@ -470,59 +693,110 @@ type FolderHierarchy = {
|
||||
parentPath?: string;
|
||||
};
|
||||
|
||||
export type ImportMarkdownZipResult = {
|
||||
docIds: string[];
|
||||
folderHierarchy?: FolderHierarchy;
|
||||
};
|
||||
|
||||
async function importMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions,
|
||||
}: ImportMarkdownZipOptions): Promise<{
|
||||
docIds: string[];
|
||||
folderHierarchy?: FolderHierarchy;
|
||||
}> {
|
||||
const provider = getProvider(extensions);
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(imported);
|
||||
}: ImportMarkdownZipOptions): Promise<ImportMarkdownZipResult> {
|
||||
return importMarkdownZipInternal({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions,
|
||||
});
|
||||
}
|
||||
|
||||
async function importNotionMarkdownZip({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions,
|
||||
}: ImportMarkdownZipOptions): Promise<ImportMarkdownZipResult> {
|
||||
return importMarkdownZipInternal({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions,
|
||||
normalizeFolderName: stripNotionHash,
|
||||
prepareMarkdownFile: prepareNotionMarkdownFile,
|
||||
preserveCommonRoot: true,
|
||||
createRootFolderForTopLevelDocs: true,
|
||||
recursiveZip: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function importMarkdownZipInternal({
|
||||
collection,
|
||||
schema,
|
||||
imported,
|
||||
extensions,
|
||||
createRootFolderForTopLevelDocs = false,
|
||||
normalizeFolderName,
|
||||
prepareMarkdownFile = prepareDefaultMarkdownFile,
|
||||
preserveCommonRoot = false,
|
||||
recursiveZip = false,
|
||||
}: ImportMarkdownZipInternalOptions): Promise<ImportMarkdownZipResult> {
|
||||
const provider = getProvider([
|
||||
markdownZipDocLinkToDeltaMatcher,
|
||||
...extensions,
|
||||
]);
|
||||
const docIds: string[] = [];
|
||||
const pendingAssets: AssetMap = new Map();
|
||||
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
|
||||
const markdownBlobs: ImportedFileEntry[] = [];
|
||||
const docPathMap: Array<{ fullPath: string; docId: string }> = [];
|
||||
const pendingPagePathIdMap = new Map<string, string>();
|
||||
const markdownBlobs: Array<ImportedFileEntry & { pageId: string }> = [];
|
||||
|
||||
// Iterate over all files in the zip
|
||||
for (const { path, content: blob } of unzip) {
|
||||
// Skip the files that are not markdown files
|
||||
if (isSystemImportPath(path)) {
|
||||
continue;
|
||||
}
|
||||
async function collectZipEntries(zipBlob: Blob, basePath = '') {
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(zipBlob);
|
||||
|
||||
// Get the file name
|
||||
const fileName = path.split('/').pop() ?? '';
|
||||
// If the file is a markdown file, store it to markdownBlobs
|
||||
if (fileName.endsWith('.md')) {
|
||||
markdownBlobs.push({
|
||||
filename: fileName,
|
||||
contentBlob: blob,
|
||||
fullPath: path,
|
||||
});
|
||||
} else {
|
||||
await stageImportedAsset({
|
||||
pendingAssets,
|
||||
pendingPathBlobIdMap,
|
||||
path,
|
||||
content: blob,
|
||||
fileName,
|
||||
});
|
||||
for (const { path, content: blob } of unzip) {
|
||||
if (isSystemImportPath(path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileName = path.split('/').pop() ?? '';
|
||||
const fullPath = basePath ? `${basePath}/${path}` : path;
|
||||
if (fileName.endsWith('.md')) {
|
||||
const pageId = collection.idGenerator();
|
||||
registerMarkdownZipPagePath(pendingPagePathIdMap, fullPath, pageId);
|
||||
markdownBlobs.push({
|
||||
filename: fileName,
|
||||
contentBlob: blob,
|
||||
fullPath,
|
||||
pageId,
|
||||
});
|
||||
} else if (recursiveZip && fileName.endsWith('.zip')) {
|
||||
await collectZipEntries(blob, getFileNameWithoutExtension(fullPath));
|
||||
} else {
|
||||
await stageImportedAsset({
|
||||
pendingAssets,
|
||||
pendingPathBlobIdMap,
|
||||
path: fullPath,
|
||||
content: blob,
|
||||
fileName,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await collectZipEntries(imported);
|
||||
|
||||
await Promise.all(
|
||||
markdownBlobs.map(async markdownFile => {
|
||||
const { filename, contentBlob, fullPath } = markdownFile;
|
||||
const fileNameWithoutExt = filename.replace(/\.[^/.]+$/, '');
|
||||
const { filename, contentBlob, fullPath, pageId } = markdownFile;
|
||||
const markdown = await contentBlob.text();
|
||||
const { content, meta } = parseFrontmatter(markdown);
|
||||
const preferredTitle = meta.title ?? fileNameWithoutExt;
|
||||
const { content, meta, preferredTitle } = prepareMarkdownFile({
|
||||
filename,
|
||||
markdown,
|
||||
});
|
||||
const job = createMarkdownImportJob({
|
||||
collection,
|
||||
schema,
|
||||
@@ -530,12 +804,15 @@ async function importMarkdownZip({
|
||||
fullPath,
|
||||
});
|
||||
bindImportedAssetsToJob(job, pendingAssets, pendingPathBlobIdMap);
|
||||
bindImportedMarkdownPagesToJob(job, pendingPagePathIdMap);
|
||||
|
||||
const mdAdapter = new MarkdownAdapter(job, provider);
|
||||
const doc = await mdAdapter.toDoc({
|
||||
const snapshot = await mdAdapter.toDocSnapshot({
|
||||
file: content,
|
||||
assets: job.assetsManager,
|
||||
});
|
||||
snapshot.meta.id = pageId;
|
||||
const doc = await job.snapshotToDoc(snapshot);
|
||||
if (doc) {
|
||||
applyMetaPatch(collection, doc.id, meta);
|
||||
docIds.push(doc.id);
|
||||
@@ -545,7 +822,12 @@ async function importMarkdownZip({
|
||||
);
|
||||
|
||||
// Build folder hierarchy from zip paths
|
||||
const folderHierarchy = buildMarkdownZipFolderHierarchy(docPathMap);
|
||||
const folderHierarchy = buildMarkdownZipFolderHierarchy(
|
||||
docPathMap,
|
||||
normalizeFolderName,
|
||||
preserveCommonRoot,
|
||||
createRootFolderForTopLevelDocs
|
||||
);
|
||||
|
||||
return { docIds, folderHierarchy };
|
||||
}
|
||||
@@ -558,15 +840,28 @@ async function importMarkdownZip({
|
||||
* hierarchy starts one level deeper.
|
||||
*/
|
||||
function buildMarkdownZipFolderHierarchy(
|
||||
entries: Array<{ fullPath: string; docId: string }>
|
||||
entries: Array<{ fullPath: string; docId: string }>,
|
||||
normalizeFolderName?: (folderName: string) => string,
|
||||
preserveCommonRoot = false,
|
||||
createRootFolderForTopLevelDocs = false
|
||||
): FolderHierarchy | undefined {
|
||||
if (entries.length === 0) return undefined;
|
||||
|
||||
// Check if any entries have folder structure
|
||||
// Check once whether all entries share a common root directory
|
||||
const candidateRoot = entries[0]?.fullPath.split('/').find(Boolean);
|
||||
const skipRoot =
|
||||
!preserveCommonRoot &&
|
||||
!!candidateRoot &&
|
||||
entries.every(e => e.fullPath.startsWith(candidateRoot + '/'));
|
||||
|
||||
// Check if any entries have folder structure after the common root is stripped.
|
||||
const hasSubfolders = entries.some(e => {
|
||||
const parts = e.fullPath.split('/').filter(Boolean);
|
||||
// More than just "root/file.md" -- need at least one real subfolder
|
||||
return parts.length > 2;
|
||||
const fileName = parts.pop();
|
||||
const folderParts = skipRoot ? parts.slice(1) : parts;
|
||||
return (
|
||||
folderParts.length > 0 || (createRootFolderForTopLevelDocs && !!fileName)
|
||||
);
|
||||
});
|
||||
if (!hasSubfolders) {
|
||||
// All files are at the same level, no folder hierarchy needed
|
||||
@@ -579,18 +874,15 @@ function buildMarkdownZipFolderHierarchy(
|
||||
children: new Map(),
|
||||
};
|
||||
|
||||
// Check once whether all entries share a common root directory
|
||||
const candidateRoot = entries[0]?.fullPath.split('/').find(Boolean);
|
||||
const skipRoot =
|
||||
!!candidateRoot &&
|
||||
entries.every(e => e.fullPath.startsWith(candidateRoot + '/'));
|
||||
|
||||
for (const { fullPath, docId } of entries) {
|
||||
const parts = fullPath.split('/').filter(Boolean);
|
||||
const fileName = parts.pop(); // Remove filename
|
||||
if (!fileName) continue;
|
||||
|
||||
let folderParts = skipRoot ? parts.slice(1) : parts;
|
||||
const folderParts = skipRoot ? parts.slice(1) : parts;
|
||||
if (folderParts.length === 0 && createRootFolderForTopLevelDocs) {
|
||||
folderParts.push(getFileNameWithoutExtension(fileName));
|
||||
}
|
||||
|
||||
if (folderParts.length === 0) {
|
||||
// Root-level file, no folder needed
|
||||
@@ -606,7 +898,7 @@ function buildMarkdownZipFolderHierarchy(
|
||||
|
||||
if (!current.children.has(folderName)) {
|
||||
current.children.set(folderName, {
|
||||
name: folderName,
|
||||
name: normalizeFolderName?.(folderName) ?? folderName,
|
||||
path: currentPath,
|
||||
parentPath: parentPath || undefined,
|
||||
children: new Map(),
|
||||
@@ -634,4 +926,5 @@ export const MarkdownTransformer = {
|
||||
importMarkdownToBlock,
|
||||
importMarkdownToDoc,
|
||||
importMarkdownZip,
|
||||
importNotionMarkdownZip,
|
||||
};
|
||||
|
||||
@@ -68,8 +68,15 @@ export class Unzip {
|
||||
|
||||
private fixFileNameEncoding(fileName: string): string {
|
||||
try {
|
||||
// check if contains non-ASCII characters
|
||||
if (fileName.split('').some(char => char.charCodeAt(0) > 127)) {
|
||||
// `fflate` already returns valid Unicode filenames for UTF-8 zip entries.
|
||||
// Only retry decoding for legacy byte-like mojibake strings, otherwise
|
||||
// normal CJK characters can be corrupted by truncating their code points.
|
||||
if (
|
||||
fileName.split('').some(char => {
|
||||
const code = char.charCodeAt(0);
|
||||
return code >= 0x80 && code <= 0xff;
|
||||
})
|
||||
) {
|
||||
// try different encodings
|
||||
const fixedName = this.tryDifferentEncodings(fileName);
|
||||
if (fixedName && fixedName !== fileName) {
|
||||
|
||||
@@ -182,4 +182,4 @@ export function createToolLoopBridge(
|
||||
|
||||
// re-export for test consumers
|
||||
export type { LlmToolCallbackRequest } from '../../../../native';
|
||||
export type { CopilotToolSet, CopilotToolExecuteOptions } from '../../tools';
|
||||
export type { CopilotToolExecuteOptions, CopilotToolSet } from '../../tools';
|
||||
|
||||
@@ -231,6 +231,7 @@ type ImportType =
|
||||
| 'markdown'
|
||||
| 'markdownZip'
|
||||
| 'notion'
|
||||
| 'notionMarkdown'
|
||||
| 'obsidian'
|
||||
| 'bear'
|
||||
| 'snapshot'
|
||||
@@ -318,6 +319,17 @@ const importOptions = [
|
||||
testId: 'editor-option-menu-import-notion',
|
||||
type: 'notion' as ImportType,
|
||||
},
|
||||
{
|
||||
key: 'notionMarkdown',
|
||||
label: 'com.affine.import.notion-markdown',
|
||||
prefixIcon: <NotionIcon color={cssVar('black')} width={20} height={20} />,
|
||||
suffixIcon: (
|
||||
<HelpIcon color={cssVarV2('icon/primary')} width={20} height={20} />
|
||||
),
|
||||
suffixTooltip: 'com.affine.import.notion-markdown.tooltip',
|
||||
testId: 'editor-option-menu-import-notion-markdown',
|
||||
type: 'notionMarkdown' as ImportType,
|
||||
},
|
||||
{
|
||||
key: 'obsidian',
|
||||
label: 'com.affine.import.obsidian',
|
||||
@@ -510,6 +522,42 @@ const importConfigs: Record<ImportType, ImportConfig> = {
|
||||
};
|
||||
},
|
||||
},
|
||||
notionMarkdown: {
|
||||
fileOptions: { acceptType: 'Zip', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
organizeService,
|
||||
explorerIconService
|
||||
) => {
|
||||
const file = files.length === 1 ? files[0] : null;
|
||||
if (!file) {
|
||||
throw new Error('Expected a single zip file for notionMarkdown import');
|
||||
}
|
||||
const { docIds, folderHierarchy } =
|
||||
await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection: docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: file,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
|
||||
const rootFolderId =
|
||||
folderHierarchy && organizeService
|
||||
? applyFolderHierarchy(
|
||||
organizeService,
|
||||
folderHierarchy,
|
||||
explorerIconService
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
docIds,
|
||||
rootFolderId,
|
||||
};
|
||||
},
|
||||
},
|
||||
obsidian: {
|
||||
fileOptions: { acceptType: 'Directory', multiple: false },
|
||||
importFunction: async (
|
||||
|
||||
@@ -2506,6 +2506,14 @@ export function useAFFiNEI18N(): {
|
||||
* `Import your Notion data. Supported import formats: HTML with subpages.`
|
||||
*/
|
||||
["com.affine.import.notion.tooltip"](): string;
|
||||
/**
|
||||
* `Notion (Markdown, .zip)`
|
||||
*/
|
||||
["com.affine.import.notion-markdown"](): string;
|
||||
/**
|
||||
* `Import a Notion Markdown export zip with subpages and attachments.`
|
||||
*/
|
||||
["com.affine.import.notion-markdown.tooltip"](): string;
|
||||
/**
|
||||
* `Obsidian Vault (Experimental)`
|
||||
*/
|
||||
|
||||
@@ -625,6 +625,8 @@
|
||||
"com.affine.import.modal.tip": "If you'd like to request support for additional file types, feel free to let us know on",
|
||||
"com.affine.import.notion": "Notion (Experimental)",
|
||||
"com.affine.import.notion.tooltip": "Import your Notion data. Supported import formats: HTML with subpages.",
|
||||
"com.affine.import.notion-markdown": "Notion (Markdown, .zip)",
|
||||
"com.affine.import.notion-markdown.tooltip": "Import a Notion Markdown export zip with subpages and attachments.",
|
||||
"com.affine.import.obsidian": "Obsidian Vault (Experimental)",
|
||||
"com.affine.import.obsidian.tooltip": "Import an Obsidian vault. Select a folder to import all notes, images, and assets with wikilinks resolved.",
|
||||
"com.affine.import.snapshot": "Snapshot",
|
||||
|
||||
Reference in New Issue
Block a user