feat(core): import progress & perf (#15197)

#### PR Dependency Tree * **PR #15197** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal)  ## Summary by CodeRabbit * **New Features** * Added a new import pipeline with “plan then commit” batch handling for Markdown, Notion HTML, Obsidian, and Bear backups. * Enabled native import sessions with progress, cancellation, and batch-by-batch committing (including assets, folders, icons, and tags). * Added web preflight limits for ZIP and multi-file imports. * **Bug Fixes** * Improved import error/warning reporting and continued processing when some items fail. * Strengthened snapshot-based file/directory picking to preserve paths. * **Chores** * Updated project packaging/configuration for the new import workflow.
2026-08-01 09:30:01 +08:00 · 2026-07-06 01:29:24 +08:00
parent 477015f064
commit 8d72e4dc29
106 changed files with 3329 additions and 11690 deletions
@@ -0,0 +1,30 @@
+import type { Framework } from '@toeverything/infra';
+
+import { ExplorerIconService } from '../explorer-icon/services/explorer-icon';
+import { OrganizeService } from '../organize';
+import { TagService } from '../tag';
+import { WorkspaceScope, WorkspaceService } from '../workspace';
+import { ImportService } from './services/service';
+
+export { ImportService };
+export type { NativeImportSessionHandlers } from './runtime-config';
+export {
+  getNativeImportSessionHandlers,
+  registerNativeImportSessionHandlers,
+} from './runtime-config';
+export type { ImportRunContext } from './services/service';
+export type {
+  NativeImportBrowserSource,
+  NativeImportFormat,
+} from '@affine/electron-api';
+
+export function configureImportModule(framework: Framework) {
+  framework
+    .scope(WorkspaceScope)
+    .service(ImportService, [
+      WorkspaceService,
+      OrganizeService,
+      ExplorerIconService,
+      TagService,
+    ]);
+}
@@ -0,0 +1,19 @@
+import type { NativeImportSessionHandlers } from '@affine/electron-api';
+
+export type {
+  NativeImportBrowserSource,
+  NativeImportFormat,
+  NativeImportSessionHandlers,
+} from '@affine/electron-api';
+
+let nativeImportSessionHandlers: NativeImportSessionHandlers | null = null;
+
+export function registerNativeImportSessionHandlers(
+  handlers: NativeImportSessionHandlers | null
+) {
+  nativeImportSessionHandlers = handlers;
+}
+
+export function getNativeImportSessionHandlers() {
+  return nativeImportSessionHandlers;
+}
@@ -0,0 +1,190 @@
+import { getStoreManager } from '@affine/core/blocksuite/manager/store';
+import { ImportCommitService } from '@affine/core/desktop/dialogs/import/commit-service';
+import { commitNativeImport } from '@affine/core/desktop/dialogs/import/native-backend';
+import {
+  preflightWebFilesImport,
+  preflightWebZipImport,
+} from '@affine/core/desktop/dialogs/import/web-limits';
+import { DebugLogger } from '@affine/debug';
+import { snapshotFile } from '@blocksuite/affine/shared/utils';
+import {
+  BearTransformer,
+  type ImportWarning,
+  MarkdownTransformer,
+  NotionHtmlTransformer,
+  ObsidianTransformer,
+  Unzip,
+} from '@blocksuite/affine/widgets/linked-doc';
+import { Service } from '@toeverything/infra';
+
+import type { ExplorerIconService } from '../../explorer-icon/services/explorer-icon';
+import type { OrganizeService } from '../../organize';
+import type { TagService } from '../../tag';
+import type { WorkspaceService } from '../../workspace';
+import { getAFFiNEWorkspaceSchema } from '../../workspace';
+
+const logger = new DebugLogger('import');
+
+export type ImportRunContext = {
+  signal?: AbortSignal;
+  onProgress?: (progress: { completed: number; total: number }) => void;
+};
+
+export class ImportService extends Service {
+  constructor(
+    private readonly workspaceService: WorkspaceService,
+    private readonly organizeService: OrganizeService,
+    private readonly explorerIconService: ExplorerIconService,
+    private readonly tagService: TagService
+  ) {
+    super();
+  }
+
+  async importMarkdownZip(file: File, context?: ImportRunContext) {
+    const collection = this.workspaceService.workspace.docCollection;
+    const commitService = this.createCommitService({ organize: true });
+    if (BUILD_CONFIG.isElectron) {
+      return commitNativeImport('markdownZip', file, commitService, context);
+    }
+
+    await preflightWebZipImport(file);
+    const snapshot = await snapshotFile(file);
+    const { batch } = await MarkdownTransformer.planMarkdownZip({
+      collection,
+      schema: getAFFiNEWorkspaceSchema(),
+      imported: snapshot,
+      extensions: getStoreManager().config.init().value.get('store'),
+    });
+    return commitService.commitBatch(batch);
+  }
+
+  async importNotionZip(file: File, context?: ImportRunContext) {
+    const collection = this.workspaceService.workspace.docCollection;
+    const commitService = this.createCommitService({
+      organize: true,
+      explorerIcon: true,
+    });
+    if (BUILD_CONFIG.isElectron) {
+      return commitNativeImport('notionZip', file, commitService, context);
+    }
+
+    await preflightWebZipImport(file);
+    const snapshot = await snapshotFile(file);
+    const format = await detectNotionZipFormat(snapshot);
+    if (format === 'markdown') {
+      const { batch } = await MarkdownTransformer.planNotionMarkdownZip({
+        collection,
+        schema: getAFFiNEWorkspaceSchema(),
+        imported: snapshot,
+        extensions: getStoreManager().config.init().value.get('store'),
+      });
+      return commitService.commitBatch(batch);
+    }
+    const { batch } = await NotionHtmlTransformer.planNotionHtmlZip({
+      collection,
+      schema: getAFFiNEWorkspaceSchema(),
+      imported: snapshot,
+      extensions: getStoreManager().config.init().value.get('store'),
+    });
+    return commitService.commitBatch(batch);
+  }
+
+  async importObsidianVault(files: File[], context?: ImportRunContext) {
+    const collection = this.workspaceService.workspace.docCollection;
+    const commitService = this.createCommitService({
+      explorerIcon: true,
+    });
+    if (!BUILD_CONFIG.isElectron) {
+      await preflightWebFilesImport(files);
+    }
+    if (BUILD_CONFIG.isElectron) {
+      return commitNativeImport('obsidian', files, commitService, context);
+    }
+    const { files: snapshots, warnings } = await snapshotReadableFiles(files);
+    if (!snapshots.length) {
+      throw new Error('No readable files were found in the selected folder.');
+    }
+
+    const { batch } = await ObsidianTransformer.planObsidianVault({
+      collection,
+      schema: getAFFiNEWorkspaceSchema(),
+      importedFiles: snapshots,
+      extensions: getStoreManager().config.init().value.get('store'),
+    });
+    batch.warnings = [...(batch.warnings ?? []), ...warnings];
+    return commitService.commitBatch(batch);
+  }
+
+  async importBearBackup(file: File, context?: ImportRunContext) {
+    const collection = this.workspaceService.workspace.docCollection;
+    const commitService = this.createCommitService({
+      organize: true,
+      tag: true,
+    });
+    if (BUILD_CONFIG.isElectron) {
+      return commitNativeImport('bearZip', file, commitService, context);
+    }
+
+    await preflightWebZipImport(file);
+    const snapshot = await snapshotFile(file);
+    const { batch } = await BearTransformer.planBearBackup({
+      collection,
+      schema: getAFFiNEWorkspaceSchema(),
+      imported: snapshot,
+      extensions: getStoreManager().config.init().value.get('store'),
+    });
+    return commitService.commitBatch(batch);
+  }
+
+  private createCommitService(options: {
+    organize?: boolean;
+    explorerIcon?: boolean;
+    tag?: boolean;
+  }) {
+    return new ImportCommitService({
+      collection: this.workspaceService.workspace.docCollection,
+      schema: getAFFiNEWorkspaceSchema(),
+      extensions: getStoreManager().config.init().value.get('store'),
+      organizeService: options.organize ? this.organizeService : undefined,
+      explorerIconService: options.explorerIcon
+        ? this.explorerIconService
+        : undefined,
+      tagService: options.tag ? this.tagService : undefined,
+      logger,
+    });
+  }
+}
+
+async function detectNotionZipFormat(file: File): Promise<'markdown' | 'html'> {
+  const unzip = new Unzip();
+  await unzip.load(file);
+  let hasHtml = false;
+  for (const entry of unzip) {
+    const lower = entry.path.toLowerCase();
+    if (lower.endsWith('.md')) return 'markdown';
+    if (lower.endsWith('.html') && !lower.endsWith('/index.html')) {
+      hasHtml = true;
+    }
+  }
+  if (hasHtml) return 'html';
+  throw new Error('No Notion Markdown or HTML pages found in the archive');
+}
+
+async function snapshotReadableFiles(files: File[]) {
+  const snapshots: File[] = [];
+  const warnings: ImportWarning[] = [];
+  for (const file of files) {
+    try {
+      snapshots.push(await snapshotFile(file));
+    } catch (error) {
+      const sourcePath = file.webkitRelativePath || file.name;
+      const reason = error instanceof Error ? error.message : String(error);
+      warnings.push({
+        code: 'file-unreadable',
+        message: `Skipped unreadable file: ${sourcePath}. ${reason}`,
+        sourcePath,
+      });
+    }
+  }
+  return { files: snapshots, warnings };
+}
@@ -34,6 +34,7 @@ import { configureFeatureFlagModule } from './feature-flag';
 import { configureGlobalContextModule } from './global-context';
 import { configureI18nModule } from './i18n';
 import { configureIconPickerModule } from './icon-picker';
+import { configureImportModule } from './import';
 import { configureImportClipperModule } from './import-clipper';
 import { configureImportTemplateModule } from './import-template';
 import { configureIntegrationModule } from './integration';
@@ -103,6 +104,7 @@ export function configureCommonModules(framework: Framework) {
  configureSystemFontFamilyModule(framework);
  configureEditorSettingModule(framework);
  configureImportTemplateModule(framework);
+  configureImportModule(framework);
  configureUserspaceModule(framework);
  configureAppSidebarModule(framework);
  configureJournalModule(framework);