mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-17 01:56:27 +08:00
feat(core): import progress & perf (#15197)
#### PR Dependency Tree * **PR #15197** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added a new import pipeline with “plan then commit” batch handling for Markdown, Notion HTML, Obsidian, and Bear backups. * Enabled native import sessions with progress, cancellation, and batch-by-batch committing (including assets, folders, icons, and tags). * Added web preflight limits for ZIP and multi-file imports. * **Bug Fixes** * Improved import error/warning reporting and continued processing when some items fail. * Strengthened snapshot-based file/directory picking to preserve paths. * **Chores** * Updated project packaging/configuration for the new import workflow. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
@@ -0,0 +1,547 @@
|
||||
import 'fake-indexeddb/auto';
|
||||
|
||||
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
|
||||
import { getAFFiNEWorkspaceSchema } from '@affine/core/modules/workspace';
|
||||
import type { DocSnapshot } from '@blocksuite/affine/store';
|
||||
import { TestWorkspace } from '@blocksuite/affine/store/test';
|
||||
import type { ImportBatch } from '@blocksuite/affine/widgets/linked-doc';
|
||||
import { describe, expect, test, vi } from 'vitest';
|
||||
|
||||
import { ImportCommitService } from './commit-service';
|
||||
|
||||
function docSnapshot(id: string, title: string): DocSnapshot {
|
||||
return {
|
||||
type: 'page',
|
||||
meta: {
|
||||
id,
|
||||
title,
|
||||
createDate: 0,
|
||||
tags: [],
|
||||
},
|
||||
blocks: {
|
||||
type: 'block',
|
||||
id: `block:${id}`,
|
||||
flavour: 'affine:page',
|
||||
props: {
|
||||
title: {
|
||||
'$blocksuite:internal:text$': true,
|
||||
delta: [{ insert: title }],
|
||||
},
|
||||
},
|
||||
children: [
|
||||
{
|
||||
type: 'block',
|
||||
id: `block:${id}:note`,
|
||||
flavour: 'affine:note',
|
||||
props: {},
|
||||
children: [
|
||||
{
|
||||
type: 'block',
|
||||
id: `block:${id}:paragraph`,
|
||||
flavour: 'affine:paragraph',
|
||||
props: {
|
||||
type: 'text',
|
||||
text: {
|
||||
'$blocksuite:internal:text$': true,
|
||||
delta: [{ insert: title }],
|
||||
},
|
||||
},
|
||||
children: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createFolderTree() {
|
||||
const folders = new Map<string, FolderNode>();
|
||||
const links: { parentId: string; docId: string }[] = [];
|
||||
let nextId = 0;
|
||||
|
||||
class FolderNode {
|
||||
readonly children: string[] = [];
|
||||
|
||||
constructor(readonly id: string) {
|
||||
folders.set(id, this);
|
||||
}
|
||||
|
||||
createFolder() {
|
||||
const id = `folder-${++nextId}`;
|
||||
this.children.push(id);
|
||||
new FolderNode(id);
|
||||
return id;
|
||||
}
|
||||
|
||||
createLink(...[, docId]: ['doc', string]) {
|
||||
links.push({ parentId: this.id, docId });
|
||||
}
|
||||
|
||||
indexAt() {
|
||||
return 'after';
|
||||
}
|
||||
}
|
||||
|
||||
const rootFolder = new FolderNode('root');
|
||||
return {
|
||||
links,
|
||||
service: {
|
||||
folderTree: {
|
||||
rootFolder,
|
||||
['folderNode$']: (id: string) => ({ value: folders.get(id) }),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createCommitService(
|
||||
collection: TestWorkspace,
|
||||
options: {
|
||||
organizeService?: unknown;
|
||||
explorerIconService?: unknown;
|
||||
tagService?: unknown;
|
||||
} = {}
|
||||
) {
|
||||
return new ImportCommitService({
|
||||
collection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
organizeService: options.organizeService as never,
|
||||
explorerIconService: options.explorerIconService as never,
|
||||
tagService: options.tagService as never,
|
||||
logger: {
|
||||
warn: vi.fn(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe('ImportCommitService', () => {
|
||||
test('commits native batch blobs, docs, folders, icons, and warnings', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
const folderTree = createFolderTree();
|
||||
const setIcon = vi.fn();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
explorerIconService: { setIcon },
|
||||
});
|
||||
|
||||
const batch: ImportBatch = {
|
||||
blobs: [
|
||||
{
|
||||
blobId: 'blob-1',
|
||||
sourcePath: 'assets/image.png',
|
||||
fileName: 'image.png',
|
||||
mime: 'image/png',
|
||||
bytes: new Uint8Array([1, 2, 3]),
|
||||
},
|
||||
],
|
||||
docs: [
|
||||
{
|
||||
id: 'doc-1',
|
||||
snapshot: docSnapshot('doc-1', 'Imported'),
|
||||
meta: { title: 'Committed title', favorite: true },
|
||||
},
|
||||
],
|
||||
folders: [
|
||||
{ path: 'root-folder', name: 'Root folder' },
|
||||
{
|
||||
path: 'root-folder/doc-1',
|
||||
name: 'Imported',
|
||||
parentPath: 'root-folder',
|
||||
pageId: 'doc-1',
|
||||
icon: { type: 'emoji', unicode: '✅' },
|
||||
},
|
||||
],
|
||||
warnings: [{ code: 'lossy', message: 'Dropped unsupported block' }],
|
||||
done: true,
|
||||
};
|
||||
|
||||
const result = await service.commitBatch(batch);
|
||||
|
||||
expect(result).toEqual({
|
||||
docIds: ['doc-1'],
|
||||
rootFolderId: 'folder-1',
|
||||
warnings: batch.warnings,
|
||||
});
|
||||
await expect(collection.blobSync.get('blob-1')).resolves.toBeInstanceOf(
|
||||
File
|
||||
);
|
||||
expect(collection.getDoc('doc-1')).not.toBeNull();
|
||||
expect(collection.meta.getDocMeta('doc-1')).toMatchObject({
|
||||
title: 'Committed title',
|
||||
favorite: true,
|
||||
});
|
||||
expect(folderTree.links).toEqual([
|
||||
{ parentId: 'folder-1', docId: 'doc-1' },
|
||||
]);
|
||||
expect(setIcon).toHaveBeenCalledWith({
|
||||
where: 'doc',
|
||||
id: 'doc-1',
|
||||
icon: { type: 'emoji', unicode: '✅' },
|
||||
});
|
||||
});
|
||||
|
||||
test('records doc commit failures as warnings and continues remaining docs', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
const service = createCommitService(collection);
|
||||
const result = await service.commitBatch({
|
||||
blobs: [],
|
||||
docs: [
|
||||
{
|
||||
id: 'doc-skip',
|
||||
sourcePath: 'docs/skip.md',
|
||||
snapshot: { ...docSnapshot('doc-skip', 'Skip'), blocks: null },
|
||||
} as never,
|
||||
{
|
||||
id: 'doc-ok',
|
||||
sourcePath: 'docs/ok.md',
|
||||
snapshot: docSnapshot('doc-ok', 'Ok'),
|
||||
meta: { title: 'Ok' },
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(result.docIds).toEqual(['doc-ok']);
|
||||
expect(result.warnings).toEqual([
|
||||
{
|
||||
code: 'skipped_doc',
|
||||
sourcePath: 'docs/skip.md',
|
||||
message:
|
||||
'Skipped docs/skip.md: document snapshot could not be committed',
|
||||
},
|
||||
]);
|
||||
expect(collection.getDoc('doc-ok')).not.toBeNull();
|
||||
});
|
||||
|
||||
test('records doc meta failures as warnings without dropping committed docs', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
const service = createCommitService(collection);
|
||||
const originalSetDocMeta = collection.meta.setDocMeta.bind(collection.meta);
|
||||
const setDocMeta = vi.spyOn(collection.meta, 'setDocMeta');
|
||||
setDocMeta.mockImplementation((id, meta) => {
|
||||
if (id === 'doc-meta') {
|
||||
throw new Error('meta failed');
|
||||
}
|
||||
return originalSetDocMeta(id, meta);
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await service.commitBatch({
|
||||
blobs: [],
|
||||
docs: [
|
||||
{
|
||||
id: 'doc-meta',
|
||||
sourcePath: 'docs/meta.md',
|
||||
snapshot: docSnapshot('doc-meta', 'Meta'),
|
||||
meta: { title: 'Meta' },
|
||||
},
|
||||
{
|
||||
id: 'doc-ok',
|
||||
sourcePath: 'docs/ok.md',
|
||||
snapshot: docSnapshot('doc-ok', 'Ok'),
|
||||
meta: { title: 'Ok' },
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(result.docIds).toEqual(['doc-meta', 'doc-ok']);
|
||||
expect(result.warnings).toEqual([
|
||||
{
|
||||
code: 'doc_meta_failed',
|
||||
sourcePath: 'docs/meta.md',
|
||||
message: 'Failed to apply metadata for docs/meta.md: meta failed',
|
||||
},
|
||||
]);
|
||||
expect(collection.getDoc('doc-meta')).not.toBeNull();
|
||||
expect(collection.getDoc('doc-ok')).not.toBeNull();
|
||||
expect(collection.meta.getDocMeta('doc-ok')).toMatchObject({
|
||||
title: 'Ok',
|
||||
});
|
||||
} finally {
|
||||
setDocMeta.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test('commits native tag names as workspace tags', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
const tags = new Map<string, { id: string; value: string }>();
|
||||
const service = createCommitService(collection, {
|
||||
tagService: {
|
||||
randomTagColor: () => 'red',
|
||||
tagList: {
|
||||
['tags$']: {
|
||||
value: [],
|
||||
},
|
||||
createTag: (value: string) => {
|
||||
const tag = { id: `tag-${tags.size + 1}`, value };
|
||||
tags.set(value, tag);
|
||||
return tag;
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await service.commitBatch({
|
||||
blobs: [],
|
||||
docs: [
|
||||
{
|
||||
id: 'doc-1',
|
||||
snapshot: docSnapshot('doc-1', 'Tagged'),
|
||||
meta: { tags: ['Blue Tag'], title: 'Tagged' },
|
||||
},
|
||||
],
|
||||
tags: [{ name: 'work/project', docIds: ['doc-1'] }],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect([...tags.values()]).toEqual([
|
||||
{ id: 'tag-1', value: 'Blue Tag' },
|
||||
{ id: 'tag-2', value: 'work' },
|
||||
]);
|
||||
expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual([
|
||||
'tag-1',
|
||||
'tag-2',
|
||||
]);
|
||||
});
|
||||
|
||||
test('commits batch folders, icons, and collapsed Bear root tags', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
collection.createDoc('doc-1');
|
||||
collection.createDoc('doc-2');
|
||||
const folderTree = createFolderTree();
|
||||
const setIcon = vi.fn();
|
||||
const tags = new Map<string, { id: string; value: string }>();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
explorerIconService: { setIcon },
|
||||
tagService: {
|
||||
randomTagColor: () => 'red',
|
||||
tagList: {
|
||||
['tags$']: {
|
||||
value: [],
|
||||
},
|
||||
createTag: (value: string) => {
|
||||
const tag = { id: `tag-${tags.size + 1}`, value };
|
||||
tags.set(value, tag);
|
||||
return tag;
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await service.commitBatch({
|
||||
docs: [],
|
||||
blobs: [],
|
||||
icons: [{ docId: 'doc-1', icon: { type: 'emoji', unicode: '📘' } }],
|
||||
tags: [
|
||||
{ name: 'work/project', docIds: ['doc-1'] },
|
||||
{ name: 'work/research', docIds: ['doc-2'] },
|
||||
],
|
||||
folders: [
|
||||
{ path: 'Bear', name: 'Bear' },
|
||||
{
|
||||
path: 'Bear/Idea',
|
||||
name: 'Idea',
|
||||
parentPath: 'Bear',
|
||||
pageId: 'doc-1',
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
docIds: [],
|
||||
rootFolderId: 'folder-1',
|
||||
warnings: [],
|
||||
});
|
||||
expect(folderTree.links).toEqual([
|
||||
{ parentId: 'folder-1', docId: 'doc-1' },
|
||||
]);
|
||||
expect(setIcon).toHaveBeenCalledWith({
|
||||
where: 'doc',
|
||||
id: 'doc-1',
|
||||
icon: { type: 'emoji', unicode: '📘' },
|
||||
});
|
||||
expect([...tags.values()]).toEqual([{ id: 'tag-1', value: 'work' }]);
|
||||
expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual(['tag-1']);
|
||||
expect(collection.meta.getDocMeta('doc-2')?.tags).toEqual(['tag-1']);
|
||||
});
|
||||
|
||||
test('keeps folder and doc links idempotent across repeated commits', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
collection.createDoc('doc-1');
|
||||
const folderTree = createFolderTree();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
});
|
||||
const batch: ImportBatch = {
|
||||
docs: [],
|
||||
blobs: [],
|
||||
folders: [
|
||||
{ path: 'Root', name: 'Root' },
|
||||
{
|
||||
path: 'Root/Doc',
|
||||
name: 'Doc',
|
||||
parentPath: 'Root',
|
||||
pageId: 'doc-1',
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
};
|
||||
|
||||
const first = await service.commitBatch(batch);
|
||||
const second = await service.commitBatch(batch);
|
||||
|
||||
expect(first.rootFolderId).toBe('folder-1');
|
||||
expect(second.rootFolderId).toBe('folder-1');
|
||||
expect(folderTree.links).toEqual([
|
||||
{ parentId: 'folder-1', docId: 'doc-1' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('resolves partial batch folder links when parent arrives later', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
collection.createDoc('doc-1');
|
||||
const folderTree = createFolderTree();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
});
|
||||
|
||||
await service.commitBatch({
|
||||
docs: [],
|
||||
blobs: [],
|
||||
folders: [
|
||||
{
|
||||
path: 'Root/Doc',
|
||||
name: 'Doc',
|
||||
parentPath: 'Root',
|
||||
pageId: 'doc-1',
|
||||
},
|
||||
],
|
||||
done: false,
|
||||
});
|
||||
await service.commitBatch({
|
||||
docs: [],
|
||||
blobs: [],
|
||||
folders: [{ path: 'Root', name: 'Root' }],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(folderTree.links).toEqual([
|
||||
{ parentId: 'folder-1', docId: 'doc-1' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('warns and clears unresolved folders on final batch', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
collection.createDoc('doc-1');
|
||||
const folderTree = createFolderTree();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
});
|
||||
|
||||
const result = await service.commitBatch({
|
||||
docs: [],
|
||||
blobs: [],
|
||||
folders: [
|
||||
{
|
||||
path: 'Missing/Doc',
|
||||
name: 'Doc',
|
||||
parentPath: 'Missing',
|
||||
pageId: 'doc-1',
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(folderTree.links).toEqual([]);
|
||||
expect(result.warnings).toEqual([
|
||||
{
|
||||
code: 'unresolved_folder',
|
||||
sourcePath: 'Missing/Doc',
|
||||
message:
|
||||
'Skipped folder placement for Missing/Doc: parent folder was not found',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('applies icons for root-level imported pages', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
collection.createDoc('doc-1');
|
||||
const folderTree = createFolderTree();
|
||||
const setIcon = vi.fn();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
explorerIconService: { setIcon },
|
||||
});
|
||||
|
||||
await service.commitBatch({
|
||||
docs: [],
|
||||
blobs: [],
|
||||
folders: [
|
||||
{
|
||||
path: 'Doc',
|
||||
name: 'Doc',
|
||||
pageId: 'doc-1',
|
||||
icon: { type: 'emoji', unicode: '📌' },
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(setIcon).toHaveBeenCalledWith({
|
||||
where: 'doc',
|
||||
id: 'doc-1',
|
||||
icon: { type: 'emoji', unicode: '📌' },
|
||||
});
|
||||
});
|
||||
|
||||
test('keeps Notion page nodes usable as child containers', async () => {
|
||||
const collection = new TestWorkspace({ id: 'test' });
|
||||
collection.meta.initialize();
|
||||
collection.createDoc('project');
|
||||
collection.createDoc('nested');
|
||||
const folderTree = createFolderTree();
|
||||
const service = createCommitService(collection, {
|
||||
organizeService: folderTree.service,
|
||||
});
|
||||
|
||||
await service.commitBatch({
|
||||
docs: [],
|
||||
blobs: [],
|
||||
folders: [
|
||||
{ path: 'Export', name: 'Export' },
|
||||
{
|
||||
path: 'Export/Project',
|
||||
name: 'Project',
|
||||
parentPath: 'Export',
|
||||
pageId: 'project',
|
||||
},
|
||||
{
|
||||
path: 'Export/Project/Nested',
|
||||
name: 'Nested',
|
||||
parentPath: 'Export/Project',
|
||||
pageId: 'nested',
|
||||
},
|
||||
],
|
||||
done: true,
|
||||
});
|
||||
|
||||
expect(folderTree.links).toEqual([
|
||||
{ parentId: 'folder-1', docId: 'project' },
|
||||
{ parentId: 'folder-2', docId: 'nested' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,333 @@
|
||||
import type { IconData } from '@affine/component';
|
||||
import type { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon';
|
||||
import type { OrganizeService } from '@affine/core/modules/organize';
|
||||
import type { TagService } from '@affine/core/modules/tag';
|
||||
import {
|
||||
type ExtensionType,
|
||||
type Schema,
|
||||
Transformer,
|
||||
type Workspace,
|
||||
} from '@blocksuite/affine/store';
|
||||
import type {
|
||||
ImportBatch,
|
||||
ImportCommitResult,
|
||||
ImportFolder,
|
||||
ImportIconData,
|
||||
} from '@blocksuite/affine/widgets/linked-doc';
|
||||
|
||||
type Logger = {
|
||||
warn: (message: string, ...args: unknown[]) => void;
|
||||
};
|
||||
|
||||
type CommitServiceOptions = {
|
||||
collection: Workspace;
|
||||
schema: Schema;
|
||||
extensions: ExtensionType[];
|
||||
organizeService?: OrganizeService;
|
||||
explorerIconService?: ExplorerIconService;
|
||||
tagService?: TagService;
|
||||
logger: Logger;
|
||||
};
|
||||
|
||||
type EmojiIconData = Extract<IconData, { unicode: string }>;
|
||||
const EMOJI_ICON_TYPE = 'emoji' as EmojiIconData['type'];
|
||||
|
||||
function copyToArrayBuffer(bytes: Uint8Array) {
|
||||
const copy = new Uint8Array(bytes.byteLength);
|
||||
copy.set(bytes);
|
||||
return copy.buffer;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown) {
|
||||
return error instanceof Error
|
||||
? error.message || error.name
|
||||
: 'Unknown error occurred';
|
||||
}
|
||||
|
||||
export class ImportCommitService {
|
||||
private readonly folderIdByPath = new Map<string, string>();
|
||||
private readonly linkedDocsByFolder = new Set<string>();
|
||||
private readonly pendingFolders: ImportFolder[] = [];
|
||||
|
||||
constructor(private readonly options: CommitServiceOptions) {}
|
||||
|
||||
async commitBatch(batch: ImportBatch): Promise<ImportCommitResult> {
|
||||
const warnings = [...(batch.warnings ?? [])];
|
||||
for (const blob of batch.blobs) {
|
||||
const bytes = new Uint8Array(blob.bytes);
|
||||
await this.options.collection.blobSync.set(
|
||||
blob.blobId,
|
||||
new File([copyToArrayBuffer(bytes)], blob.fileName, {
|
||||
type: blob.mime,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
const transformer = new Transformer({
|
||||
schema: this.options.schema,
|
||||
blobCRUD: this.options.collection.blobSync,
|
||||
docCRUD: {
|
||||
create: (id: string) =>
|
||||
this.options.collection.createDoc(id).getStore({ id }),
|
||||
get: (id: string) =>
|
||||
this.options.collection.getDoc(id)?.getStore({ id }) ?? null,
|
||||
delete: (id: string) => this.options.collection.removeDoc(id),
|
||||
},
|
||||
middlewares: [],
|
||||
});
|
||||
|
||||
const docIds: string[] = [];
|
||||
const tags = new Map<string, string[]>();
|
||||
for (const doc of batch.docs) {
|
||||
let store: Awaited<ReturnType<Transformer['snapshotToDoc']>>;
|
||||
try {
|
||||
store = await transformer.snapshotToDoc(doc.snapshot);
|
||||
} catch (error) {
|
||||
warnings.push({
|
||||
code: 'skipped_doc',
|
||||
sourcePath: doc.sourcePath,
|
||||
message: `Skipped ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!store) {
|
||||
warnings.push({
|
||||
code: 'skipped_doc',
|
||||
sourcePath: doc.sourcePath,
|
||||
message: `Skipped ${doc.sourcePath ?? doc.id}: document snapshot could not be committed`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
docIds.push(store.id);
|
||||
if (doc.meta && Object.keys(doc.meta).length) {
|
||||
try {
|
||||
const { tags: docTags, ...meta } = doc.meta;
|
||||
if (Object.keys(meta).length) {
|
||||
this.options.collection.meta.setDocMeta(store.id, meta);
|
||||
}
|
||||
for (const tag of docTags ?? []) {
|
||||
const docIds = tags.get(tag) ?? [];
|
||||
docIds.push(store.id);
|
||||
tags.set(tag, docIds);
|
||||
}
|
||||
} catch (error) {
|
||||
warnings.push({
|
||||
code: 'doc_meta_failed',
|
||||
sourcePath: doc.sourcePath,
|
||||
message: `Failed to apply metadata for ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const tag of batch.tags ?? []) {
|
||||
const docIds = tags.get(tag.name) ?? [];
|
||||
docIds.push(...tag.docIds);
|
||||
tags.set(tag.name, docIds);
|
||||
}
|
||||
|
||||
const rootFolderId = this.applyNativeFolders(
|
||||
batch.folders ?? [],
|
||||
warnings,
|
||||
batch.done
|
||||
);
|
||||
this.applyNativeTags(tags);
|
||||
this.applyNativeIcons(batch.icons);
|
||||
return {
|
||||
docIds,
|
||||
entryId: batch.entryId,
|
||||
isWorkspaceFile: batch.isWorkspaceFile,
|
||||
rootFolderId,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
private applyNativeFolders(
|
||||
folders: ImportFolder[],
|
||||
warnings: ImportCommitResult['warnings'],
|
||||
batchDone: boolean
|
||||
): string | undefined {
|
||||
const { organizeService } = this.options;
|
||||
if (folders.length === 0) return undefined;
|
||||
if (!organizeService) {
|
||||
for (const folder of folders) {
|
||||
if (folder.pageId && !folder.parentPath) {
|
||||
this.applyIcon(folder.pageId, folder.icon);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
let rootFolderId: string | undefined;
|
||||
this.pendingFolders.push(...folders);
|
||||
let progressed = true;
|
||||
|
||||
while (progressed && this.pendingFolders.length) {
|
||||
progressed = false;
|
||||
const nextPending: ImportFolder[] = [];
|
||||
const pending = this.pendingFolders.splice(0);
|
||||
const childParentPaths = new Set(
|
||||
pending
|
||||
.map(folder => folder.parentPath)
|
||||
.filter((path): path is string => !!path)
|
||||
);
|
||||
for (const folder of pending) {
|
||||
const needsContainer =
|
||||
!folder.pageId || childParentPaths.has(folder.path);
|
||||
if (needsContainer && !this.folderIdByPath.has(folder.path)) {
|
||||
const parent = folder.parentPath
|
||||
? this.folderIdByPath.has(folder.parentPath)
|
||||
? organizeService.folderTree.folderNode$(
|
||||
this.folderIdByPath.get(folder.parentPath) ?? ''
|
||||
).value
|
||||
: null
|
||||
: organizeService.folderTree.rootFolder;
|
||||
if (!parent) {
|
||||
nextPending.push(folder);
|
||||
continue;
|
||||
}
|
||||
const folderId = parent.createFolder(
|
||||
folder.name,
|
||||
parent.indexAt('after')
|
||||
);
|
||||
this.folderIdByPath.set(folder.path, folderId);
|
||||
rootFolderId ??= folderId;
|
||||
progressed = true;
|
||||
} else if (needsContainer) {
|
||||
rootFolderId ??= this.folderIdByPath.get(folder.path);
|
||||
}
|
||||
|
||||
if (folder.pageId) {
|
||||
if (!this.applyFolderDocLink(folder)) {
|
||||
nextPending.push(folder);
|
||||
continue;
|
||||
}
|
||||
progressed = true;
|
||||
}
|
||||
}
|
||||
if (progressed && nextPending.length) {
|
||||
this.pendingFolders.push(...nextPending);
|
||||
} else if (batchDone) {
|
||||
for (const folder of nextPending) {
|
||||
warnings.push({
|
||||
code: 'unresolved_folder',
|
||||
sourcePath: folder.path,
|
||||
message: `Skipped folder placement for ${folder.path}: parent folder was not found`,
|
||||
});
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
this.pendingFolders.push(...nextPending);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return rootFolderId;
|
||||
} catch (error) {
|
||||
this.options.logger.warn('Failed to commit import folders:', error);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private applyFolderDocLink(folder: ImportFolder): boolean {
|
||||
const { organizeService } = this.options;
|
||||
if (!folder.pageId) return true;
|
||||
if (!folder.parentPath) {
|
||||
this.applyIcon(folder.pageId, folder.icon);
|
||||
return true;
|
||||
}
|
||||
if (!organizeService) return true;
|
||||
const parentFolderId = this.folderIdByPath.get(folder.parentPath);
|
||||
if (!parentFolderId) return false;
|
||||
const linkKey = `${parentFolderId}:${folder.pageId}`;
|
||||
if (!this.linkedDocsByFolder.has(linkKey)) {
|
||||
const parent =
|
||||
organizeService.folderTree.folderNode$(parentFolderId).value;
|
||||
if (!parent) return false;
|
||||
parent.createLink('doc', folder.pageId, parent.indexAt('after'));
|
||||
this.linkedDocsByFolder.add(linkKey);
|
||||
}
|
||||
this.applyIcon(folder.pageId, folder.icon);
|
||||
return true;
|
||||
}
|
||||
|
||||
private applyNativeIcons(icons?: ImportBatch['icons']) {
|
||||
for (const icon of icons ?? []) {
|
||||
this.applyIcon(icon.docId, icon.icon);
|
||||
}
|
||||
}
|
||||
|
||||
private applyIcon(id: string, icon?: ImportIconData) {
|
||||
if (!icon || !this.options.explorerIconService) return;
|
||||
const iconData = toIconData(icon);
|
||||
if (!iconData) return;
|
||||
this.options.explorerIconService.setIcon({
|
||||
where: 'doc',
|
||||
id,
|
||||
icon: iconData,
|
||||
});
|
||||
}
|
||||
|
||||
private applyNativeTags(tags?: Map<string, string[]>) {
|
||||
const { tagService, collection } = this.options;
|
||||
if (!tagService || !tags?.size) return;
|
||||
|
||||
try {
|
||||
const existingTagMap = new Map<string, string>();
|
||||
for (const tag of tagService.tagList.tags$.value) {
|
||||
existingTagMap.set(tag.value$.value.toLowerCase(), tag.id);
|
||||
}
|
||||
|
||||
const rootTagDocMap = new Map<
|
||||
string,
|
||||
{ displayName: string; docs: Set<string> }
|
||||
>();
|
||||
for (const [tagName, tagDocIds] of tags) {
|
||||
const originalRoot = tagName.split('/')[0];
|
||||
const key = originalRoot.toLowerCase();
|
||||
let entry = rootTagDocMap.get(key);
|
||||
if (!entry) {
|
||||
entry = { displayName: originalRoot, docs: new Set() };
|
||||
rootTagDocMap.set(key, entry);
|
||||
}
|
||||
for (const docId of tagDocIds) {
|
||||
entry.docs.add(docId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [rootTagKey, { displayName, docs }] of rootTagDocMap) {
|
||||
let tagId = existingTagMap.get(rootTagKey);
|
||||
if (!tagId) {
|
||||
const newTag = tagService.tagList.createTag(
|
||||
displayName,
|
||||
tagService.randomTagColor()
|
||||
);
|
||||
tagId = newTag.id;
|
||||
existingTagMap.set(rootTagKey, tagId);
|
||||
}
|
||||
|
||||
for (const docId of docs) {
|
||||
const doc = collection.getDoc(docId);
|
||||
const currentTags = doc?.meta?.tags ?? [];
|
||||
if (!currentTags.includes(tagId)) {
|
||||
collection.meta.setDocMeta(docId, {
|
||||
tags: [...currentTags, tagId],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.options.logger.warn('Failed to commit import tags:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function toIconData(icon: ImportIconData): IconData | undefined {
|
||||
if (icon.type === 'emoji') {
|
||||
return {
|
||||
type: EMOJI_ICON_TYPE,
|
||||
unicode: icon.unicode,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,10 +1,4 @@
|
||||
import {
|
||||
Button,
|
||||
IconButton,
|
||||
type IconData,
|
||||
IconType,
|
||||
Modal,
|
||||
} from '@affine/component';
|
||||
import { Button, IconButton, Modal } from '@affine/component';
|
||||
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
|
||||
import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks';
|
||||
import { useNavigateHelper } from '@affine/core/components/hooks/use-navigate-helper';
|
||||
@@ -13,9 +7,10 @@ import {
|
||||
GlobalDialogService,
|
||||
type WORKSPACE_DIALOG_SCHEMA,
|
||||
} from '@affine/core/modules/dialogs';
|
||||
import { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon';
|
||||
import { OrganizeService } from '@affine/core/modules/organize';
|
||||
import { TagService } from '@affine/core/modules/tag';
|
||||
import {
|
||||
type ImportRunContext,
|
||||
ImportService,
|
||||
} from '@affine/core/modules/import';
|
||||
import { UrlService } from '@affine/core/modules/url';
|
||||
import {
|
||||
getAFFiNEWorkspaceSchema,
|
||||
@@ -28,12 +23,10 @@ import track from '@affine/track';
|
||||
import { openDirectory, openFilesWith } from '@blocksuite/affine/shared/utils';
|
||||
import type { Workspace } from '@blocksuite/affine/store';
|
||||
import {
|
||||
BearTransformer,
|
||||
DocxTransformer,
|
||||
HtmlTransformer,
|
||||
type ImportWarning,
|
||||
MarkdownTransformer,
|
||||
NotionHtmlTransformer,
|
||||
ObsidianTransformer,
|
||||
ZipTransformer,
|
||||
} from '@blocksuite/affine/widgets/linked-doc';
|
||||
import {
|
||||
@@ -54,6 +47,7 @@ import {
|
||||
type SVGAttributes,
|
||||
useCallback,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from 'react';
|
||||
|
||||
@@ -61,177 +55,15 @@ import * as style from './styles.css';
|
||||
|
||||
const logger = new DebugLogger('import');
|
||||
|
||||
type NotionPageIcon = {
|
||||
type: 'emoji' | 'image';
|
||||
content: string; // emoji unicode or image URL/data
|
||||
};
|
||||
|
||||
type FolderHierarchy = {
|
||||
name: string;
|
||||
path: string;
|
||||
children: Map<string, FolderHierarchy>;
|
||||
pageId?: string;
|
||||
parentPath?: string;
|
||||
icon?: NotionPageIcon;
|
||||
};
|
||||
|
||||
// Helper function to create folder structure using OrganizeService
|
||||
function createFolderStructure(
|
||||
organizeService: OrganizeService,
|
||||
hierarchy: FolderHierarchy,
|
||||
parentFolderId: string | null = null,
|
||||
explorerIconService?: ExplorerIconService
|
||||
): {
|
||||
folderId: string | null;
|
||||
docLinks: Array<{ folderId: string; docId: string }>;
|
||||
} {
|
||||
const docLinks: Array<{ folderId: string; docId: string }> = [];
|
||||
const rootFolder = organizeService.folderTree.rootFolder;
|
||||
|
||||
function processHierarchyNode(
|
||||
node: FolderHierarchy,
|
||||
currentParentId: string | null
|
||||
): string | null {
|
||||
let currentFolderId = currentParentId;
|
||||
|
||||
// If this node represents a folder (has children but no pageId), create it
|
||||
if (node.children.size > 0 && !node.pageId && node.name) {
|
||||
const parent = currentParentId
|
||||
? organizeService.folderTree.folderNode$(currentParentId).value
|
||||
: rootFolder;
|
||||
|
||||
if (parent) {
|
||||
const index = parent.indexAt('after');
|
||||
currentFolderId = parent.createFolder(node.name, index);
|
||||
}
|
||||
}
|
||||
|
||||
// Process all children
|
||||
for (const child of node.children.values()) {
|
||||
if (child.pageId) {
|
||||
// This is a document, link it to the current folder
|
||||
if (currentFolderId) {
|
||||
docLinks.push({ folderId: currentFolderId, docId: child.pageId });
|
||||
}
|
||||
|
||||
// Set icon for the document if available
|
||||
if (child.icon && explorerIconService) {
|
||||
logger.debug('=== Setting icon for document ===');
|
||||
logger.debug('Document ID:', child.pageId);
|
||||
logger.debug('Icon data:', child.icon);
|
||||
|
||||
try {
|
||||
let iconData: IconData | undefined;
|
||||
if (child.icon.type === 'emoji') {
|
||||
iconData = {
|
||||
type: IconType.Emoji,
|
||||
unicode: child.icon.content,
|
||||
};
|
||||
logger.debug('Created emoji icon data:', iconData);
|
||||
} else if (child.icon.type === 'image') {
|
||||
// For image icons, we'd need to handle blob conversion
|
||||
// For now, let's skip image icons or convert them to default
|
||||
// This could be enhanced later to download and convert images to blobs
|
||||
logger.debug(
|
||||
'Skipping image icon (not implemented):',
|
||||
child.icon.content
|
||||
);
|
||||
iconData = undefined;
|
||||
}
|
||||
|
||||
if (iconData) {
|
||||
logger.debug('Calling explorerIconService.setIcon with:', {
|
||||
where: 'doc',
|
||||
id: child.pageId,
|
||||
icon: iconData,
|
||||
});
|
||||
explorerIconService.setIcon({
|
||||
where: 'doc',
|
||||
id: child.pageId,
|
||||
icon: iconData,
|
||||
});
|
||||
logger.debug('Icon set successfully for document:', child.pageId);
|
||||
} else {
|
||||
logger.debug('No valid icon data to set');
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
'Error setting icon for document:',
|
||||
child.pageId,
|
||||
error
|
||||
);
|
||||
logger.warn(
|
||||
'Failed to set icon for document:',
|
||||
child.pageId,
|
||||
error
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if (!child.icon) {
|
||||
logger.debug('No icon found for document:', child.pageId);
|
||||
}
|
||||
if (!explorerIconService) {
|
||||
logger.debug(
|
||||
'ExplorerIconService not available for document:',
|
||||
child.pageId
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (child.children.size > 0) {
|
||||
// This is a subfolder, process it recursively
|
||||
processHierarchyNode(child, currentFolderId);
|
||||
}
|
||||
}
|
||||
|
||||
return currentFolderId;
|
||||
}
|
||||
|
||||
const rootFolderId = processHierarchyNode(hierarchy, parentFolderId);
|
||||
return { folderId: rootFolderId, docLinks };
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the folder tree described by {@link folderHierarchy} via
|
||||
* {@link OrganizeService} and links every document into its folder.
|
||||
* Returns the root folder ID on success, or `undefined` if the
|
||||
* hierarchy is empty or an error occurs.
|
||||
*
|
||||
* When {@link explorerIconService} is provided, document icons from the
|
||||
* hierarchy (e.g. Notion page emojis) are applied. Callers that do not
|
||||
* need icon support can omit it safely.
|
||||
*/
|
||||
function applyFolderHierarchy(
|
||||
organizeService: OrganizeService,
|
||||
folderHierarchy: FolderHierarchy,
|
||||
explorerIconService?: ExplorerIconService
|
||||
): string | undefined {
|
||||
if (folderHierarchy.children.size === 0) return undefined;
|
||||
try {
|
||||
const { folderId, docLinks } = createFolderStructure(
|
||||
organizeService,
|
||||
folderHierarchy,
|
||||
null,
|
||||
explorerIconService
|
||||
);
|
||||
for (const { folderId, docId } of docLinks) {
|
||||
const folder = organizeService.folderTree.folderNode$(folderId).value;
|
||||
if (folder) {
|
||||
const index = folder.indexAt('after');
|
||||
folder.createLink('doc', docId, index);
|
||||
}
|
||||
}
|
||||
return folderId || undefined;
|
||||
} catch (error) {
|
||||
logger.warn('Failed to create folder structure:', error);
|
||||
return undefined;
|
||||
}
|
||||
function shouldSnapshotPickedFiles(type: ImportType, acceptType: AcceptType) {
|
||||
if (acceptType === 'Directory' || acceptType === 'Skip') return false;
|
||||
return !['markdownZip', 'notion', 'bear'].includes(type);
|
||||
}
|
||||
|
||||
type ImportType =
|
||||
| 'markdown'
|
||||
| 'markdownZip'
|
||||
| 'notion'
|
||||
| 'notionMarkdown'
|
||||
| 'obsidian'
|
||||
| 'bear'
|
||||
| 'snapshot'
|
||||
@@ -240,28 +72,71 @@ type ImportType =
|
||||
| 'dotaffinefile';
|
||||
type AcceptType = 'Markdown' | 'Zip' | 'Html' | 'Docx' | 'Directory' | 'Skip'; // Skip is used for dotaffinefile
|
||||
type Status = 'idle' | 'importing' | 'success' | 'error';
|
||||
type ImportErrorState = {
|
||||
code: string;
|
||||
message: string;
|
||||
sourcePath?: string;
|
||||
};
|
||||
type ImportResult = {
|
||||
docIds: string[];
|
||||
entryId?: string;
|
||||
isWorkspaceFile?: boolean;
|
||||
rootFolderId?: string;
|
||||
importedWorkspace?: WorkspaceMetadata;
|
||||
warnings?: ImportWarning[];
|
||||
};
|
||||
|
||||
type ImportedWorkspacePayload = {
|
||||
workspace: WorkspaceMetadata;
|
||||
};
|
||||
|
||||
type ImportFunctionArgs = {
|
||||
docCollection: Workspace;
|
||||
files: File[];
|
||||
importAffineFile: () => Promise<WorkspaceMetadata | undefined>;
|
||||
importService?: ImportService;
|
||||
context: ImportRunContext;
|
||||
};
|
||||
|
||||
function toImportErrorState(error: unknown): ImportErrorState {
|
||||
const sourcePath =
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'sourcePath' in error &&
|
||||
typeof error.sourcePath === 'string'
|
||||
? error.sourcePath
|
||||
: undefined;
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return {
|
||||
code: 'cancelled',
|
||||
message: 'Import cancelled',
|
||||
sourcePath,
|
||||
};
|
||||
}
|
||||
if (error instanceof Error) {
|
||||
return {
|
||||
code: error.name || 'import-error',
|
||||
message: error.message || 'Unknown error occurred',
|
||||
sourcePath,
|
||||
};
|
||||
}
|
||||
return {
|
||||
code: 'unknown',
|
||||
message: 'Unknown error occurred',
|
||||
sourcePath,
|
||||
};
|
||||
}
|
||||
|
||||
function requireImportService(importService?: ImportService) {
|
||||
if (!importService) {
|
||||
throw new Error('Import service is unavailable');
|
||||
}
|
||||
return importService;
|
||||
}
|
||||
|
||||
type ImportConfig = {
|
||||
fileOptions: { acceptType: AcceptType; multiple: boolean };
|
||||
importFunction: (
|
||||
docCollection: Workspace,
|
||||
files: File[],
|
||||
handleImportAffineFile: () => Promise<WorkspaceMetadata | undefined>,
|
||||
organizeService?: OrganizeService,
|
||||
explorerIconService?: ExplorerIconService,
|
||||
tagService?: TagService
|
||||
) => Promise<ImportResult>;
|
||||
importFunction: (args: ImportFunctionArgs) => Promise<ImportResult>;
|
||||
};
|
||||
|
||||
const importOptions = [
|
||||
@@ -319,17 +194,6 @@ const importOptions = [
|
||||
testId: 'editor-option-menu-import-notion',
|
||||
type: 'notion' as ImportType,
|
||||
},
|
||||
{
|
||||
key: 'notionMarkdown',
|
||||
label: 'com.affine.import.notion-markdown',
|
||||
prefixIcon: <NotionIcon color={cssVar('black')} width={20} height={20} />,
|
||||
suffixIcon: (
|
||||
<HelpIcon color={cssVarV2('icon/primary')} width={20} height={20} />
|
||||
),
|
||||
suffixTooltip: 'com.affine.import.notion-markdown.tooltip',
|
||||
testId: 'editor-option-menu-import-notion-markdown',
|
||||
type: 'notionMarkdown' as ImportType,
|
||||
},
|
||||
{
|
||||
key: 'obsidian',
|
||||
label: 'com.affine.import.obsidian',
|
||||
@@ -400,13 +264,7 @@ const importOptions = [
|
||||
const importConfigs: Record<ImportType, ImportConfig> = {
|
||||
markdown: {
|
||||
fileOptions: { acceptType: 'Markdown', multiple: true },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
_organizeService,
|
||||
_explorerIconService
|
||||
) => {
|
||||
importFunction: async ({ docCollection, files }) => {
|
||||
const docIds: string[] = [];
|
||||
for (const file of files) {
|
||||
const text = await file.text();
|
||||
@@ -427,45 +285,20 @@ const importConfigs: Record<ImportType, ImportConfig> = {
|
||||
},
|
||||
markdownZip: {
|
||||
fileOptions: { acceptType: 'Zip', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
organizeService,
|
||||
_explorerIconService
|
||||
) => {
|
||||
importFunction: async ({ files, importService, context }) => {
|
||||
const file = files.length === 1 ? files[0] : null;
|
||||
if (!file) {
|
||||
throw new Error('Expected a single zip file for markdownZip import');
|
||||
}
|
||||
const { docIds, folderHierarchy } =
|
||||
await MarkdownTransformer.importMarkdownZip({
|
||||
collection: docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: file,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
|
||||
const rootFolderId =
|
||||
folderHierarchy && organizeService
|
||||
? applyFolderHierarchy(organizeService, folderHierarchy)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
docIds,
|
||||
rootFolderId,
|
||||
};
|
||||
return requireImportService(importService).importMarkdownZip(
|
||||
file,
|
||||
context
|
||||
);
|
||||
},
|
||||
},
|
||||
html: {
|
||||
fileOptions: { acceptType: 'Html', multiple: true },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
_organizeService,
|
||||
_explorerIconService
|
||||
) => {
|
||||
importFunction: async ({ docCollection, files }) => {
|
||||
const docIds: string[] = [];
|
||||
for (const file of files) {
|
||||
const text = await file.text();
|
||||
@@ -486,220 +319,39 @@ const importConfigs: Record<ImportType, ImportConfig> = {
|
||||
},
|
||||
notion: {
|
||||
fileOptions: { acceptType: 'Zip', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
organizeService,
|
||||
explorerIconService
|
||||
) => {
|
||||
importFunction: async ({ files, importService, context }) => {
|
||||
const file = files.length === 1 ? files[0] : null;
|
||||
if (!file) {
|
||||
throw new Error('Expected a single zip file for notion import');
|
||||
}
|
||||
const { entryId, pageIds, isWorkspaceFile, folderHierarchy } =
|
||||
await NotionHtmlTransformer.importNotionZip({
|
||||
collection: docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: file,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
|
||||
const rootFolderId =
|
||||
folderHierarchy && organizeService
|
||||
? applyFolderHierarchy(
|
||||
organizeService,
|
||||
folderHierarchy,
|
||||
explorerIconService
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
docIds: pageIds,
|
||||
entryId,
|
||||
isWorkspaceFile,
|
||||
rootFolderId,
|
||||
};
|
||||
},
|
||||
},
|
||||
notionMarkdown: {
|
||||
fileOptions: { acceptType: 'Zip', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
organizeService,
|
||||
explorerIconService
|
||||
) => {
|
||||
const file = files.length === 1 ? files[0] : null;
|
||||
if (!file) {
|
||||
throw new Error('Expected a single zip file for notionMarkdown import');
|
||||
}
|
||||
const { docIds, folderHierarchy } =
|
||||
await MarkdownTransformer.importNotionMarkdownZip({
|
||||
collection: docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: file,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
|
||||
const rootFolderId =
|
||||
folderHierarchy && organizeService
|
||||
? applyFolderHierarchy(
|
||||
organizeService,
|
||||
folderHierarchy,
|
||||
explorerIconService
|
||||
)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
docIds,
|
||||
rootFolderId,
|
||||
};
|
||||
return requireImportService(importService).importNotionZip(file, context);
|
||||
},
|
||||
},
|
||||
obsidian: {
|
||||
fileOptions: { acceptType: 'Directory', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
_organizeService,
|
||||
explorerIconService
|
||||
) => {
|
||||
const { docIds, docEmojis } =
|
||||
await ObsidianTransformer.importObsidianVault({
|
||||
collection: docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
importedFiles: files,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
|
||||
if (explorerIconService) {
|
||||
for (const [id, emoji] of docEmojis.entries()) {
|
||||
explorerIconService.setIcon({
|
||||
where: 'doc',
|
||||
id,
|
||||
icon: { type: IconType.Emoji, unicode: emoji },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { docIds };
|
||||
importFunction: async ({ files, importService, context }) => {
|
||||
return requireImportService(importService).importObsidianVault(
|
||||
files,
|
||||
context
|
||||
);
|
||||
},
|
||||
},
|
||||
bear: {
|
||||
fileOptions: { acceptType: 'Zip', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
organizeService,
|
||||
_explorerIconService,
|
||||
tagService
|
||||
) => {
|
||||
importFunction: async ({ files, importService, context }) => {
|
||||
const file = files.length === 1 ? files[0] : null;
|
||||
if (!file) {
|
||||
throw new Error('Expected a single .bear2bk file for Bear import');
|
||||
}
|
||||
let docIds: string[];
|
||||
let tags: Map<string, string[]>;
|
||||
let folderHierarchy: FolderHierarchy;
|
||||
try {
|
||||
const result = await BearTransformer.importBearBackup({
|
||||
collection: docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: file,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
docIds = result.docIds;
|
||||
tags = result.tags;
|
||||
folderHierarchy = result.folderHierarchy;
|
||||
} catch (err) {
|
||||
logger.error('Bear import failed:', err);
|
||||
throw err instanceof Error
|
||||
? err
|
||||
: new Error(String(err) || 'Bear import failed');
|
||||
}
|
||||
|
||||
// Create AFFiNE tags from Bear tags
|
||||
if (tagService && tags.size > 0) {
|
||||
try {
|
||||
// Get existing tags for deduplication
|
||||
const existingTags = tagService.tagList.tags$.value;
|
||||
const existingTagMap = new Map<string, string>(); // lowercase name → tag id
|
||||
for (const tag of existingTags) {
|
||||
const name = tag.value$.value.toLowerCase();
|
||||
existingTagMap.set(name, tag.id);
|
||||
}
|
||||
|
||||
// Consolidate tags by root segment (e.g., "privat/bike" → "privat").
|
||||
// Keyed by lowercase root for case-insensitive dedup, but the
|
||||
// original capitalization of the first occurrence is preserved
|
||||
// so new AFFiNE tags are created with the user's casing.
|
||||
const rootTagDocMap = new Map<
|
||||
string,
|
||||
{ displayName: string; docs: Set<string> }
|
||||
>();
|
||||
for (const [tagName, tagDocIds] of tags) {
|
||||
const originalRoot = tagName.split('/')[0];
|
||||
const key = originalRoot.toLowerCase();
|
||||
let entry = rootTagDocMap.get(key);
|
||||
if (!entry) {
|
||||
entry = { displayName: originalRoot, docs: new Set<string>() };
|
||||
rootTagDocMap.set(key, entry);
|
||||
}
|
||||
for (const docId of tagDocIds) {
|
||||
entry.docs.add(docId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [
|
||||
rootTagKey,
|
||||
{ displayName, docs: docIdSet },
|
||||
] of rootTagDocMap) {
|
||||
// Check if tag already exists (case-insensitive)
|
||||
let tagId = existingTagMap.get(rootTagKey);
|
||||
if (!tagId) {
|
||||
const newTag = tagService.tagList.createTag(
|
||||
displayName,
|
||||
tagService.randomTagColor()
|
||||
);
|
||||
tagId = newTag.id;
|
||||
existingTagMap.set(rootTagKey, tagId);
|
||||
}
|
||||
|
||||
// Assign tag to each doc
|
||||
for (const docId of docIdSet) {
|
||||
const doc = docCollection.getDoc(docId);
|
||||
const currentTags = doc?.meta?.tags ?? [];
|
||||
if (!currentTags.includes(tagId)) {
|
||||
docCollection.meta.setDocMeta(docId, {
|
||||
tags: [...currentTags, tagId],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to create Bear tags:', error);
|
||||
}
|
||||
}
|
||||
|
||||
const rootFolderId =
|
||||
folderHierarchy && organizeService
|
||||
? applyFolderHierarchy(organizeService, folderHierarchy)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
docIds,
|
||||
rootFolderId,
|
||||
};
|
||||
return requireImportService(importService).importBearBackup(
|
||||
file,
|
||||
context
|
||||
);
|
||||
},
|
||||
},
|
||||
docx: {
|
||||
fileOptions: { acceptType: 'Docx', multiple: false },
|
||||
importFunction: async (docCollection, file) => {
|
||||
const files = Array.isArray(file) ? file : [file];
|
||||
importFunction: async ({ docCollection, files }) => {
|
||||
const docIds: string[] = [];
|
||||
for (const file of files) {
|
||||
const docId = await DocxTransformer.importDocx({
|
||||
@@ -715,13 +367,7 @@ const importConfigs: Record<ImportType, ImportConfig> = {
|
||||
},
|
||||
snapshot: {
|
||||
fileOptions: { acceptType: 'Zip', multiple: false },
|
||||
importFunction: async (
|
||||
docCollection,
|
||||
files,
|
||||
_handleImportAffineFile,
|
||||
_organizeService,
|
||||
_explorerIconService
|
||||
) => {
|
||||
importFunction: async ({ docCollection, files }) => {
|
||||
const file = files.length === 1 ? files[0] : null;
|
||||
if (!file) {
|
||||
throw new Error('Expected a single zip file for snapshot import');
|
||||
@@ -743,14 +389,8 @@ const importConfigs: Record<ImportType, ImportConfig> = {
|
||||
},
|
||||
dotaffinefile: {
|
||||
fileOptions: { acceptType: 'Skip', multiple: false },
|
||||
importFunction: async (
|
||||
_,
|
||||
__,
|
||||
handleImportAffineFile,
|
||||
_organizeService,
|
||||
_explorerIconService
|
||||
) => {
|
||||
const workspace = await handleImportAffineFile();
|
||||
importFunction: async ({ importAffineFile }) => {
|
||||
const workspace = await importAffineFile();
|
||||
return {
|
||||
docIds: [],
|
||||
entryId: undefined,
|
||||
@@ -843,8 +483,18 @@ const ImportOptions = ({
|
||||
);
|
||||
};
|
||||
|
||||
const ImportingStatus = () => {
|
||||
const ImportingStatus = ({
|
||||
progress,
|
||||
onCancel,
|
||||
}: {
|
||||
progress: { completed: number; total: number } | null;
|
||||
onCancel: () => void;
|
||||
}) => {
|
||||
const t = useI18n();
|
||||
const progressLabel =
|
||||
progress && progress.total > 0
|
||||
? `${progress.completed}/${progress.total}`
|
||||
: null;
|
||||
return (
|
||||
<>
|
||||
<div className={style.importModalTitle}>
|
||||
@@ -853,11 +503,25 @@ const ImportingStatus = () => {
|
||||
<p className={style.importStatusContent}>
|
||||
{t['com.affine.import.status.importing.message']()}
|
||||
</p>
|
||||
{progressLabel ? (
|
||||
<div className={style.importProgress}>{progressLabel}</div>
|
||||
) : null}
|
||||
<div className={style.importModalButtonContainer}>
|
||||
<Button onClick={onCancel} variant="secondary">
|
||||
{t['Cancel']()}
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => {
|
||||
const SuccessStatus = ({
|
||||
warnings,
|
||||
onComplete,
|
||||
}: {
|
||||
warnings: string[];
|
||||
onComplete: () => void;
|
||||
}) => {
|
||||
const t = useI18n();
|
||||
return (
|
||||
<>
|
||||
@@ -876,6 +540,13 @@ const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => {
|
||||
</a>
|
||||
.
|
||||
</p>
|
||||
{warnings.length ? (
|
||||
<div className={style.importWarnings}>
|
||||
{warnings.map((warning, index) => (
|
||||
<div key={`${warning}-${index}`}>{warning}</div>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
<div className={style.importModalButtonContainer}>
|
||||
<Button onClick={onComplete} variant="primary">
|
||||
{t['Complete']()}
|
||||
@@ -889,7 +560,7 @@ const ErrorStatus = ({
|
||||
error,
|
||||
onRetry,
|
||||
}: {
|
||||
error: string | null;
|
||||
error: ImportErrorState | null;
|
||||
onRetry: () => void;
|
||||
}) => {
|
||||
const t = useI18n();
|
||||
@@ -900,8 +571,11 @@ const ErrorStatus = ({
|
||||
{t['com.affine.import.status.failed.title']()}
|
||||
</div>
|
||||
<p className={style.importStatusContent}>
|
||||
{error || 'Unknown error occurred'}
|
||||
{error?.message || 'Unknown error occurred'}
|
||||
</p>
|
||||
{error?.sourcePath ? (
|
||||
<div className={style.importErrorDetail}>{error.sourcePath}</div>
|
||||
) : null}
|
||||
<div className={style.importModalButtonContainer}>
|
||||
<Button
|
||||
onClick={() => {
|
||||
@@ -924,13 +598,16 @@ export const ImportDialog = ({
|
||||
}: DialogComponentProps<WORKSPACE_DIALOG_SCHEMA['import']>) => {
|
||||
const t = useI18n();
|
||||
const [status, setStatus] = useState<Status>('idle');
|
||||
const [importError, setImportError] = useState<string | null>(null);
|
||||
const [importError, setImportError] = useState<ImportErrorState | null>(null);
|
||||
const [importResult, setImportResult] = useState<ImportResult | null>(null);
|
||||
const [importProgress, setImportProgress] = useState<{
|
||||
completed: number;
|
||||
total: number;
|
||||
} | null>(null);
|
||||
const importAbortControllerRef = useRef<AbortController | null>(null);
|
||||
const workspace = useService(WorkspaceService).workspace;
|
||||
const docCollection = workspace.docCollection;
|
||||
const organizeService = useService(OrganizeService);
|
||||
const explorerIconService = useService(ExplorerIconService);
|
||||
const tagService = useService(TagService);
|
||||
const importService = useService(ImportService);
|
||||
|
||||
const globalDialogService = useService(GlobalDialogService);
|
||||
|
||||
@@ -984,6 +661,7 @@ export const ImportDialog = ({
|
||||
const handleImport = useAsyncCallback(
|
||||
async (type: ImportType) => {
|
||||
setImportError(null);
|
||||
setImportProgress(null);
|
||||
try {
|
||||
const importConfig = importConfigs[type];
|
||||
const { acceptType, multiple } = importConfig.fileOptions;
|
||||
@@ -992,8 +670,13 @@ export const ImportDialog = ({
|
||||
acceptType === 'Skip'
|
||||
? []
|
||||
: acceptType === 'Directory'
|
||||
? await openDirectory()
|
||||
: await openFilesWith(acceptType, multiple);
|
||||
? await openDirectory({
|
||||
fileSystemAccess: false,
|
||||
})
|
||||
: await openFilesWith(acceptType, multiple, {
|
||||
fileSystemAccess: false,
|
||||
snapshot: shouldSnapshotPickedFiles(type, acceptType),
|
||||
});
|
||||
|
||||
if (!files || (files.length === 0 && acceptType !== 'Skip')) {
|
||||
throw new Error(
|
||||
@@ -1009,20 +692,28 @@ export const ImportDialog = ({
|
||||
});
|
||||
}
|
||||
|
||||
const abortController = new AbortController();
|
||||
importAbortControllerRef.current = abortController;
|
||||
const {
|
||||
docIds,
|
||||
entryId,
|
||||
isWorkspaceFile,
|
||||
rootFolderId,
|
||||
importedWorkspace,
|
||||
} = await importConfig.importFunction(
|
||||
warnings,
|
||||
} = await importConfig.importFunction({
|
||||
docCollection,
|
||||
files,
|
||||
handleImportAffineFile,
|
||||
organizeService,
|
||||
explorerIconService,
|
||||
tagService
|
||||
);
|
||||
importAffineFile: handleImportAffineFile,
|
||||
importService,
|
||||
context: {
|
||||
signal: abortController.signal,
|
||||
onProgress: progress => {
|
||||
setImportProgress(progress);
|
||||
},
|
||||
},
|
||||
});
|
||||
importAbortControllerRef.current = null;
|
||||
|
||||
setImportResult({
|
||||
docIds,
|
||||
@@ -1030,6 +721,7 @@ export const ImportDialog = ({
|
||||
isWorkspaceFile,
|
||||
rootFolderId,
|
||||
importedWorkspace,
|
||||
warnings,
|
||||
});
|
||||
setStatus('success');
|
||||
track.$.importModal.$.import({
|
||||
@@ -1043,26 +735,19 @@ export const ImportDialog = ({
|
||||
control: 'import',
|
||||
});
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : 'Unknown error occurred';
|
||||
setImportError(errorMessage);
|
||||
importAbortControllerRef.current = null;
|
||||
const structuredError = toImportErrorState(error);
|
||||
setImportError(structuredError);
|
||||
setStatus('error');
|
||||
track.$.importModal.$.import({
|
||||
type,
|
||||
status: 'failed',
|
||||
error: errorMessage || undefined,
|
||||
error: structuredError.message || undefined,
|
||||
});
|
||||
logger.error('Failed to import', error);
|
||||
}
|
||||
},
|
||||
[
|
||||
docCollection,
|
||||
explorerIconService,
|
||||
handleImportAffineFile,
|
||||
organizeService,
|
||||
tagService,
|
||||
t,
|
||||
]
|
||||
[docCollection, handleImportAffineFile, importService, t]
|
||||
);
|
||||
|
||||
const finishImport = useCallback(() => {
|
||||
@@ -1073,8 +758,11 @@ export const ImportDialog = ({
|
||||
close();
|
||||
return;
|
||||
}
|
||||
const { importedWorkspace: _workspace, ...result } = importResult;
|
||||
close(result);
|
||||
close({
|
||||
docIds: importResult.docIds,
|
||||
entryId: importResult.entryId,
|
||||
isWorkspaceFile: importResult.isWorkspaceFile,
|
||||
});
|
||||
}, [close, handleCreatedWorkspace, importResult]);
|
||||
|
||||
const handleComplete = useCallback(() => {
|
||||
@@ -1082,13 +770,27 @@ export const ImportDialog = ({
|
||||
}, [finishImport]);
|
||||
|
||||
const handleRetry = () => {
|
||||
setImportProgress(null);
|
||||
setStatus('idle');
|
||||
};
|
||||
|
||||
const handleCancel = useCallback(() => {
|
||||
importAbortControllerRef.current?.abort();
|
||||
}, []);
|
||||
|
||||
const statusComponents = {
|
||||
idle: <ImportOptions onImport={handleImport} />,
|
||||
importing: <ImportingStatus />,
|
||||
success: <SuccessStatus onComplete={handleComplete} />,
|
||||
importing: (
|
||||
<ImportingStatus progress={importProgress} onCancel={handleCancel} />
|
||||
),
|
||||
success: (
|
||||
<SuccessStatus
|
||||
warnings={(importResult?.warnings ?? []).map(warning =>
|
||||
typeof warning === 'string' ? warning : warning.message
|
||||
)}
|
||||
onComplete={handleComplete}
|
||||
/>
|
||||
),
|
||||
error: <ErrorStatus error={importError} onRetry={handleRetry} />,
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import {
|
||||
getNativeImportSessionHandlers,
|
||||
type NativeImportSessionHandlers,
|
||||
} from '@affine/core/modules/import/runtime-config';
|
||||
import type {
|
||||
NativeImportBrowserSource,
|
||||
NativeImportFormat,
|
||||
} from '@affine/electron-api';
|
||||
import type {
|
||||
ImportBatch,
|
||||
ImportCommitResult,
|
||||
} from '@blocksuite/affine/widgets/linked-doc';
|
||||
|
||||
import type { ImportCommitService } from './commit-service';
|
||||
|
||||
const NATIVE_IMPORT_BATCH_LIMITS = {
|
||||
maxDocs: 20,
|
||||
maxBlobs: 20,
|
||||
maxBlobBytes: 10 * 1024 * 1024,
|
||||
};
|
||||
|
||||
export async function commitNativeImport(
|
||||
format: NativeImportFormat,
|
||||
source: File | File[],
|
||||
commitService: ImportCommitService,
|
||||
options: {
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (progress: { completed: number; total: number }) => void;
|
||||
} = {}
|
||||
): Promise<ImportCommitResult> {
|
||||
const native = await getNativeImporter();
|
||||
if (options.signal?.aborted) {
|
||||
throw new DOMException('Import cancelled', 'AbortError');
|
||||
}
|
||||
const sessionId = await native.createImportSession({
|
||||
format,
|
||||
source: getNativeImportSource(source),
|
||||
batchLimits: NATIVE_IMPORT_BATCH_LIMITS,
|
||||
});
|
||||
|
||||
const docIds: string[] = [];
|
||||
const warnings: ImportCommitResult['warnings'] = [];
|
||||
let entryId: string | undefined;
|
||||
let isWorkspaceFile = false;
|
||||
let rootFolderId: string | undefined;
|
||||
let cancelled = false;
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
if (options.signal?.aborted) {
|
||||
cancelled = true;
|
||||
try {
|
||||
await native.cancelImportSession(sessionId);
|
||||
} catch {
|
||||
// Preserve the AbortError reported to callers.
|
||||
}
|
||||
throw new DOMException('Import cancelled', 'AbortError');
|
||||
}
|
||||
const payload = await native.nextImportBatch(sessionId);
|
||||
if (!payload) break;
|
||||
const batch = JSON.parse(payload) as ImportBatch;
|
||||
options.onProgress?.(batch.progress ?? { completed: 0, total: 0 });
|
||||
const result = await commitService.commitBatch(batch);
|
||||
docIds.push(...result.docIds);
|
||||
warnings.push(...result.warnings);
|
||||
entryId ??= batch.entryId;
|
||||
isWorkspaceFile ||= !!batch.isWorkspaceFile;
|
||||
rootFolderId ??= result.rootFolderId;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
try {
|
||||
await native.cancelImportSession(sessionId);
|
||||
} catch {
|
||||
// Preserve the original import error.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
await native.disposeImportSession(sessionId);
|
||||
}
|
||||
|
||||
if (!docIds.length && !isWorkspaceFile) {
|
||||
throw new Error('No importable documents were found in the selected file.');
|
||||
}
|
||||
|
||||
return {
|
||||
docIds,
|
||||
entryId,
|
||||
isWorkspaceFile,
|
||||
rootFolderId,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
async function getNativeImporter(): Promise<NativeImportSessionHandlers> {
|
||||
const registered = getNativeImportSessionHandlers();
|
||||
if (registered) {
|
||||
return registered;
|
||||
}
|
||||
throw new Error('Native import session handlers are not registered');
|
||||
}
|
||||
|
||||
function getNativeImportSource(
|
||||
source: File | File[]
|
||||
): NativeImportBrowserSource {
|
||||
if (Array.isArray(source)) {
|
||||
return { kind: 'directory', files: source };
|
||||
}
|
||||
return { kind: 'file', file: source };
|
||||
}
|
||||
@@ -57,6 +57,30 @@ export const importStatusContent = style({
|
||||
color: cssVar('textPrimaryColor'),
|
||||
});
|
||||
|
||||
export const importProgress = style({
|
||||
width: '100%',
|
||||
fontSize: cssVar('fontSm'),
|
||||
lineHeight: cssVar('lineHeight'),
|
||||
color: cssVar('textSecondaryColor'),
|
||||
});
|
||||
|
||||
export const importWarnings = style({
|
||||
width: '100%',
|
||||
maxHeight: '120px',
|
||||
overflow: 'auto',
|
||||
fontSize: cssVar('fontSm'),
|
||||
lineHeight: cssVar('lineHeight'),
|
||||
color: cssVar('textSecondaryColor'),
|
||||
});
|
||||
|
||||
export const importErrorDetail = style({
|
||||
width: '100%',
|
||||
fontSize: cssVar('fontXs'),
|
||||
lineHeight: cssVar('lineHeight'),
|
||||
color: cssVar('textSecondaryColor'),
|
||||
overflowWrap: 'anywhere',
|
||||
});
|
||||
|
||||
export const importModalButtonContainer = style({
|
||||
width: '100%',
|
||||
display: 'flex',
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, expect, test } from 'vitest';
|
||||
|
||||
import {
|
||||
preflightWebFilesImport,
|
||||
preflightWebZipImport,
|
||||
WebImportLimitError,
|
||||
} from './web-limits';
|
||||
|
||||
function zipDirectoryFile(names: string[]) {
|
||||
const chunks = names.map(name => {
|
||||
const nameBytes = new TextEncoder().encode(name);
|
||||
const header = new Uint8Array(46 + nameBytes.length);
|
||||
writeU32(header, 0, 0x02014b50);
|
||||
writeU32(header, 20, 1);
|
||||
writeU32(header, 24, 1);
|
||||
writeU16(header, 28, nameBytes.length);
|
||||
header.set(nameBytes, 46);
|
||||
return header;
|
||||
});
|
||||
return new File(chunks, 'import.zip', { type: 'application/zip' });
|
||||
}
|
||||
|
||||
function writeU16(bytes: Uint8Array, offset: number, value: number) {
|
||||
bytes[offset] = value & 0xff;
|
||||
bytes[offset + 1] = (value >> 8) & 0xff;
|
||||
}
|
||||
|
||||
function writeU32(bytes: Uint8Array, offset: number, value: number) {
|
||||
bytes[offset] = value & 0xff;
|
||||
bytes[offset + 1] = (value >> 8) & 0xff;
|
||||
bytes[offset + 2] = (value >> 16) & 0xff;
|
||||
bytes[offset + 3] = (value >> 24) & 0xff;
|
||||
}
|
||||
|
||||
describe('preflightWebZipImport', () => {
|
||||
test('rejects nested zip before JS importer parsing', async () => {
|
||||
await expect(
|
||||
preflightWebZipImport(zipDirectoryFile(['entry.md', 'nested.zip']))
|
||||
).rejects.toBeInstanceOf(WebImportLimitError);
|
||||
});
|
||||
|
||||
test('rejects document count over web limit', async () => {
|
||||
await expect(
|
||||
preflightWebZipImport(zipDirectoryFile(['a.md', 'b.md']), {
|
||||
maxTotalBytes: 1024,
|
||||
maxEntryBytes: 1024,
|
||||
maxEntryCount: 10,
|
||||
maxNestedZipDepth: 0,
|
||||
maxDocumentCount: 1,
|
||||
})
|
||||
).rejects.toBeInstanceOf(WebImportLimitError);
|
||||
});
|
||||
|
||||
test('counts html pages as web import documents', async () => {
|
||||
await expect(
|
||||
preflightWebZipImport(zipDirectoryFile(['a.html', 'b.htm']), {
|
||||
maxTotalBytes: 1024,
|
||||
maxEntryBytes: 1024,
|
||||
maxEntryCount: 10,
|
||||
maxNestedZipDepth: 0,
|
||||
maxDocumentCount: 1,
|
||||
})
|
||||
).rejects.toBeInstanceOf(WebImportLimitError);
|
||||
});
|
||||
|
||||
test('allows small zip directory', async () => {
|
||||
await expect(
|
||||
preflightWebZipImport(zipDirectoryFile(['entry.md']), {
|
||||
maxTotalBytes: 1024,
|
||||
maxEntryBytes: 1024,
|
||||
maxEntryCount: 10,
|
||||
maxNestedZipDepth: 0,
|
||||
maxDocumentCount: 1,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('preflightWebFilesImport', () => {
|
||||
test('rejects directory import over web document limit', async () => {
|
||||
await expect(
|
||||
preflightWebFilesImport(
|
||||
[new File(['a'], 'a.md'), new File(['b'], 'b.md')],
|
||||
{
|
||||
maxTotalBytes: 1024,
|
||||
maxEntryBytes: 1024,
|
||||
maxEntryCount: 10,
|
||||
maxNestedZipDepth: 0,
|
||||
maxDocumentCount: 1,
|
||||
}
|
||||
)
|
||||
).rejects.toBeInstanceOf(WebImportLimitError);
|
||||
});
|
||||
|
||||
test('allows small directory import', async () => {
|
||||
await expect(
|
||||
preflightWebFilesImport([new File(['a'], 'a.md')], {
|
||||
maxTotalBytes: 1024,
|
||||
maxEntryBytes: 1024,
|
||||
maxEntryCount: 10,
|
||||
maxNestedZipDepth: 0,
|
||||
maxDocumentCount: 1,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
export type WebImportLimits = {
|
||||
maxTotalBytes: number;
|
||||
maxEntryBytes: number;
|
||||
maxEntryCount: number;
|
||||
maxNestedZipDepth: number;
|
||||
maxDocumentCount: number;
|
||||
};
|
||||
|
||||
export const webImportLimits: WebImportLimits = {
|
||||
maxTotalBytes: 32 * 1024 * 1024,
|
||||
maxEntryBytes: 8 * 1024 * 1024,
|
||||
maxEntryCount: 1000,
|
||||
maxNestedZipDepth: 0,
|
||||
maxDocumentCount: 250,
|
||||
};
|
||||
|
||||
export class WebImportLimitError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'WebImportLimitError';
|
||||
}
|
||||
}
|
||||
|
||||
export async function preflightWebZipImport(
|
||||
file: File,
|
||||
limits = webImportLimits
|
||||
) {
|
||||
if (file.size > limits.maxTotalBytes) {
|
||||
throw new WebImportLimitError(
|
||||
'This import is too large for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
|
||||
const entries = readZipDirectory(new Uint8Array(await file.arrayBuffer()));
|
||||
if (entries.length > limits.maxEntryCount) {
|
||||
throw new WebImportLimitError(
|
||||
'This import has too many files for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
|
||||
const documentCount = entries.filter(entry =>
|
||||
isWebImportDocument(entry.name)
|
||||
).length;
|
||||
if (documentCount > limits.maxDocumentCount) {
|
||||
throw new WebImportLimitError(
|
||||
'This import has too many documents for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.uncompressedSize > limits.maxEntryBytes) {
|
||||
throw new WebImportLimitError(
|
||||
'This import contains a file that is too large for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
if (
|
||||
limits.maxNestedZipDepth === 0 &&
|
||||
entry.name.toLowerCase().endsWith('.zip')
|
||||
) {
|
||||
throw new WebImportLimitError(
|
||||
'This import contains a nested zip. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function preflightWebFilesImport(
|
||||
files: File[],
|
||||
limits = webImportLimits
|
||||
) {
|
||||
const totalBytes = files.reduce((total, file) => total + file.size, 0);
|
||||
if (totalBytes > limits.maxTotalBytes) {
|
||||
throw new WebImportLimitError(
|
||||
'This import is too large for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
if (files.length > limits.maxEntryCount) {
|
||||
throw new WebImportLimitError(
|
||||
'This import has too many files for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
|
||||
let documentCount = 0;
|
||||
for (const file of files) {
|
||||
if (file.size > limits.maxEntryBytes) {
|
||||
throw new WebImportLimitError(
|
||||
'This import contains a file that is too large for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
const path = file.webkitRelativePath || file.name;
|
||||
if (path.toLowerCase().endsWith('.zip')) {
|
||||
throw new WebImportLimitError(
|
||||
'This import contains a nested zip. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
if (isWebImportDocument(path)) {
|
||||
documentCount += 1;
|
||||
}
|
||||
}
|
||||
if (documentCount > limits.maxDocumentCount) {
|
||||
throw new WebImportLimitError(
|
||||
'This import has too many documents for the web app. Please import it in the desktop client.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isWebImportDocument(path: string) {
|
||||
const lower = path.toLowerCase();
|
||||
return (
|
||||
lower.endsWith('.md') || lower.endsWith('.html') || lower.endsWith('.htm')
|
||||
);
|
||||
}
|
||||
|
||||
type ZipEntry = {
|
||||
name: string;
|
||||
uncompressedSize: number;
|
||||
};
|
||||
|
||||
function readZipDirectory(bytes: Uint8Array): ZipEntry[] {
|
||||
const entries: ZipEntry[] = [];
|
||||
for (let offset = 0; offset + 46 <= bytes.length; offset++) {
|
||||
if (readU32(bytes, offset) !== 0x02014b50) continue;
|
||||
|
||||
const compressedSize = readU32(bytes, offset + 20);
|
||||
const uncompressedSize = readU32(bytes, offset + 24);
|
||||
const fileNameLength = readU16(bytes, offset + 28);
|
||||
const extraLength = readU16(bytes, offset + 30);
|
||||
const commentLength = readU16(bytes, offset + 32);
|
||||
const nameStart = offset + 46;
|
||||
const nameEnd = nameStart + fileNameLength;
|
||||
if (nameEnd > bytes.length) break;
|
||||
|
||||
entries.push({
|
||||
name: new TextDecoder().decode(bytes.subarray(nameStart, nameEnd)),
|
||||
uncompressedSize: uncompressedSize || compressedSize,
|
||||
});
|
||||
offset = nameEnd + extraLength + commentLength - 1;
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
function readU16(bytes: Uint8Array, offset: number) {
|
||||
return bytes[offset] | (bytes[offset + 1] << 8);
|
||||
}
|
||||
|
||||
function readU32(bytes: Uint8Array, offset: number) {
|
||||
return (
|
||||
(bytes[offset] |
|
||||
(bytes[offset + 1] << 8) |
|
||||
(bytes[offset + 2] << 16) |
|
||||
(bytes[offset + 3] << 24)) >>>
|
||||
0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { Framework } from '@toeverything/infra';
|
||||
|
||||
import { ExplorerIconService } from '../explorer-icon/services/explorer-icon';
|
||||
import { OrganizeService } from '../organize';
|
||||
import { TagService } from '../tag';
|
||||
import { WorkspaceScope, WorkspaceService } from '../workspace';
|
||||
import { ImportService } from './services/service';
|
||||
|
||||
export { ImportService };
|
||||
export type { NativeImportSessionHandlers } from './runtime-config';
|
||||
export {
|
||||
getNativeImportSessionHandlers,
|
||||
registerNativeImportSessionHandlers,
|
||||
} from './runtime-config';
|
||||
export type { ImportRunContext } from './services/service';
|
||||
export type {
|
||||
NativeImportBrowserSource,
|
||||
NativeImportFormat,
|
||||
} from '@affine/electron-api';
|
||||
|
||||
export function configureImportModule(framework: Framework) {
|
||||
framework
|
||||
.scope(WorkspaceScope)
|
||||
.service(ImportService, [
|
||||
WorkspaceService,
|
||||
OrganizeService,
|
||||
ExplorerIconService,
|
||||
TagService,
|
||||
]);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { NativeImportSessionHandlers } from '@affine/electron-api';
|
||||
|
||||
export type {
|
||||
NativeImportBrowserSource,
|
||||
NativeImportFormat,
|
||||
NativeImportSessionHandlers,
|
||||
} from '@affine/electron-api';
|
||||
|
||||
let nativeImportSessionHandlers: NativeImportSessionHandlers | null = null;
|
||||
|
||||
export function registerNativeImportSessionHandlers(
|
||||
handlers: NativeImportSessionHandlers | null
|
||||
) {
|
||||
nativeImportSessionHandlers = handlers;
|
||||
}
|
||||
|
||||
export function getNativeImportSessionHandlers() {
|
||||
return nativeImportSessionHandlers;
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
|
||||
import { ImportCommitService } from '@affine/core/desktop/dialogs/import/commit-service';
|
||||
import { commitNativeImport } from '@affine/core/desktop/dialogs/import/native-backend';
|
||||
import {
|
||||
preflightWebFilesImport,
|
||||
preflightWebZipImport,
|
||||
} from '@affine/core/desktop/dialogs/import/web-limits';
|
||||
import { DebugLogger } from '@affine/debug';
|
||||
import { snapshotFile } from '@blocksuite/affine/shared/utils';
|
||||
import {
|
||||
BearTransformer,
|
||||
type ImportWarning,
|
||||
MarkdownTransformer,
|
||||
NotionHtmlTransformer,
|
||||
ObsidianTransformer,
|
||||
Unzip,
|
||||
} from '@blocksuite/affine/widgets/linked-doc';
|
||||
import { Service } from '@toeverything/infra';
|
||||
|
||||
import type { ExplorerIconService } from '../../explorer-icon/services/explorer-icon';
|
||||
import type { OrganizeService } from '../../organize';
|
||||
import type { TagService } from '../../tag';
|
||||
import type { WorkspaceService } from '../../workspace';
|
||||
import { getAFFiNEWorkspaceSchema } from '../../workspace';
|
||||
|
||||
const logger = new DebugLogger('import');
|
||||
|
||||
export type ImportRunContext = {
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (progress: { completed: number; total: number }) => void;
|
||||
};
|
||||
|
||||
export class ImportService extends Service {
|
||||
constructor(
|
||||
private readonly workspaceService: WorkspaceService,
|
||||
private readonly organizeService: OrganizeService,
|
||||
private readonly explorerIconService: ExplorerIconService,
|
||||
private readonly tagService: TagService
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async importMarkdownZip(file: File, context?: ImportRunContext) {
|
||||
const collection = this.workspaceService.workspace.docCollection;
|
||||
const commitService = this.createCommitService({ organize: true });
|
||||
if (BUILD_CONFIG.isElectron) {
|
||||
return commitNativeImport('markdownZip', file, commitService, context);
|
||||
}
|
||||
|
||||
await preflightWebZipImport(file);
|
||||
const snapshot = await snapshotFile(file);
|
||||
const { batch } = await MarkdownTransformer.planMarkdownZip({
|
||||
collection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: snapshot,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
return commitService.commitBatch(batch);
|
||||
}
|
||||
|
||||
async importNotionZip(file: File, context?: ImportRunContext) {
|
||||
const collection = this.workspaceService.workspace.docCollection;
|
||||
const commitService = this.createCommitService({
|
||||
organize: true,
|
||||
explorerIcon: true,
|
||||
});
|
||||
if (BUILD_CONFIG.isElectron) {
|
||||
return commitNativeImport('notionZip', file, commitService, context);
|
||||
}
|
||||
|
||||
await preflightWebZipImport(file);
|
||||
const snapshot = await snapshotFile(file);
|
||||
const format = await detectNotionZipFormat(snapshot);
|
||||
if (format === 'markdown') {
|
||||
const { batch } = await MarkdownTransformer.planNotionMarkdownZip({
|
||||
collection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: snapshot,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
return commitService.commitBatch(batch);
|
||||
}
|
||||
const { batch } = await NotionHtmlTransformer.planNotionHtmlZip({
|
||||
collection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: snapshot,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
return commitService.commitBatch(batch);
|
||||
}
|
||||
|
||||
async importObsidianVault(files: File[], context?: ImportRunContext) {
|
||||
const collection = this.workspaceService.workspace.docCollection;
|
||||
const commitService = this.createCommitService({
|
||||
explorerIcon: true,
|
||||
});
|
||||
if (!BUILD_CONFIG.isElectron) {
|
||||
await preflightWebFilesImport(files);
|
||||
}
|
||||
if (BUILD_CONFIG.isElectron) {
|
||||
return commitNativeImport('obsidian', files, commitService, context);
|
||||
}
|
||||
const { files: snapshots, warnings } = await snapshotReadableFiles(files);
|
||||
if (!snapshots.length) {
|
||||
throw new Error('No readable files were found in the selected folder.');
|
||||
}
|
||||
|
||||
const { batch } = await ObsidianTransformer.planObsidianVault({
|
||||
collection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
importedFiles: snapshots,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
batch.warnings = [...(batch.warnings ?? []), ...warnings];
|
||||
return commitService.commitBatch(batch);
|
||||
}
|
||||
|
||||
async importBearBackup(file: File, context?: ImportRunContext) {
|
||||
const collection = this.workspaceService.workspace.docCollection;
|
||||
const commitService = this.createCommitService({
|
||||
organize: true,
|
||||
tag: true,
|
||||
});
|
||||
if (BUILD_CONFIG.isElectron) {
|
||||
return commitNativeImport('bearZip', file, commitService, context);
|
||||
}
|
||||
|
||||
await preflightWebZipImport(file);
|
||||
const snapshot = await snapshotFile(file);
|
||||
const { batch } = await BearTransformer.planBearBackup({
|
||||
collection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
imported: snapshot,
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
});
|
||||
return commitService.commitBatch(batch);
|
||||
}
|
||||
|
||||
private createCommitService(options: {
|
||||
organize?: boolean;
|
||||
explorerIcon?: boolean;
|
||||
tag?: boolean;
|
||||
}) {
|
||||
return new ImportCommitService({
|
||||
collection: this.workspaceService.workspace.docCollection,
|
||||
schema: getAFFiNEWorkspaceSchema(),
|
||||
extensions: getStoreManager().config.init().value.get('store'),
|
||||
organizeService: options.organize ? this.organizeService : undefined,
|
||||
explorerIconService: options.explorerIcon
|
||||
? this.explorerIconService
|
||||
: undefined,
|
||||
tagService: options.tag ? this.tagService : undefined,
|
||||
logger,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function detectNotionZipFormat(file: File): Promise<'markdown' | 'html'> {
|
||||
const unzip = new Unzip();
|
||||
await unzip.load(file);
|
||||
let hasHtml = false;
|
||||
for (const entry of unzip) {
|
||||
const lower = entry.path.toLowerCase();
|
||||
if (lower.endsWith('.md')) return 'markdown';
|
||||
if (lower.endsWith('.html') && !lower.endsWith('/index.html')) {
|
||||
hasHtml = true;
|
||||
}
|
||||
}
|
||||
if (hasHtml) return 'html';
|
||||
throw new Error('No Notion Markdown or HTML pages found in the archive');
|
||||
}
|
||||
|
||||
async function snapshotReadableFiles(files: File[]) {
|
||||
const snapshots: File[] = [];
|
||||
const warnings: ImportWarning[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
snapshots.push(await snapshotFile(file));
|
||||
} catch (error) {
|
||||
const sourcePath = file.webkitRelativePath || file.name;
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
warnings.push({
|
||||
code: 'file-unreadable',
|
||||
message: `Skipped unreadable file: ${sourcePath}. ${reason}`,
|
||||
sourcePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { files: snapshots, warnings };
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import { configureFeatureFlagModule } from './feature-flag';
|
||||
import { configureGlobalContextModule } from './global-context';
|
||||
import { configureI18nModule } from './i18n';
|
||||
import { configureIconPickerModule } from './icon-picker';
|
||||
import { configureImportModule } from './import';
|
||||
import { configureImportClipperModule } from './import-clipper';
|
||||
import { configureImportTemplateModule } from './import-template';
|
||||
import { configureIntegrationModule } from './integration';
|
||||
@@ -103,6 +104,7 @@ export function configureCommonModules(framework: Framework) {
|
||||
configureSystemFontFamilyModule(framework);
|
||||
configureEditorSettingModule(framework);
|
||||
configureImportTemplateModule(framework);
|
||||
configureImportModule(framework);
|
||||
configureUserspaceModule(framework);
|
||||
configureAppSidebarModule(framework);
|
||||
configureJournalModule(framework);
|
||||
|
||||
Reference in New Issue
Block a user