feat(core): import progress & perf (#15197)

#### PR Dependency Tree


* **PR #15197** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a new import pipeline with “plan then commit” batch handling for
Markdown, Notion HTML, Obsidian, and Bear backups.
* Enabled native import sessions with progress, cancellation, and
batch-by-batch committing (including assets, folders, icons, and tags).
  * Added web preflight limits for ZIP and multi-file imports.
* **Bug Fixes**
* Improved import error/warning reporting and continued processing when
some items fail.
* Strengthened snapshot-based file/directory picking to preserve paths.
* **Chores**
  * Updated project packaging/configuration for the new import workflow.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-06 01:29:24 +08:00
committed by GitHub
parent 477015f064
commit 8d72e4dc29
106 changed files with 3329 additions and 11690 deletions
@@ -0,0 +1,15 @@
import { registerNativeImportSessionHandlers } from '@affine/core/modules/import';
import { apis } from '@affine/electron-api';
const importApis = apis?.import;
if (importApis) {
registerNativeImportSessionHandlers({
createImportSession: options =>
importApis.createImportSessionFromSource(options),
nextImportBatch: sessionId => importApis.nextImportBatch(sessionId),
cancelImportSession: sessionId => importApis.cancelImportSession(sessionId),
disposeImportSession: sessionId =>
importApis.disposeImportSession(sessionId),
});
}
@@ -2,6 +2,7 @@ import '@affine/core/bootstrap/electron';
import '@affine/core/bootstrap/cleanup';
import '@affine/component/theme';
import './global.css';
import './native-import';
import { apis } from '@affine/electron-api';
import { bindNativeDBApis } from '@affine/nbstore/sqlite';
@@ -15,6 +15,7 @@
"./main/exposed": "./src/main/exposed.ts",
"./preload/electron-api": "./src/preload/electron-api.ts",
"./preload/shared-storage": "./src/preload/shared-storage.ts",
"./shared/import": "./src/shared/import.ts",
"./main/shared-state-schema": "./src/main/shared-state-schema.ts",
"./main/updater/event": "./src/main/updater/event.ts",
"./main/windows-manager": "./src/main/windows-manager/index.ts"
@@ -7,6 +7,7 @@ import { byokStorageHandlers } from './byok-storage/handlers';
import { clipboardHandlers } from './clipboard';
import { configStorageHandlers } from './config-storage';
import { findInPageHandlers } from './find-in-page';
import { importHandlers } from './import';
import { getLogFilePath, logger, revealLogFile } from './logger';
import { recordingHandlers } from './recording';
import { checkSource } from './security-restrictions';
@@ -39,6 +40,7 @@ export const allHandlers = {
updater: updaterHandlers,
configStorage: configStorageHandlers,
findInPage: findInPageHandlers,
import: importHandlers,
sharedStorage: sharedStorageHandlers,
worker: workerHandlers,
recording: recordingHandlers,
@@ -0,0 +1,39 @@
import type { CreateImportSessionOptions } from '@affine/native';
import {
cancelImportSession,
createImportSession,
disposeImportSession,
nextImportBatch,
} from '@affine/native';
export const importHandlers = {
createImportSession: (
event: Electron.IpcMainInvokeEvent,
options: CreateImportSessionOptions
) => {
void event;
return createImportSession({
format: options.format,
source: options.source,
batchLimits: options.batchLimits,
});
},
nextImportBatch: (event: Electron.IpcMainInvokeEvent, sessionId: string) => {
void event;
return nextImportBatch(sessionId);
},
cancelImportSession: (
event: Electron.IpcMainInvokeEvent,
sessionId: string
) => {
void event;
return cancelImportSession(sessionId);
},
disposeImportSession: (
event: Electron.IpcMainInvokeEvent,
sessionId: string
) => {
void event;
return disposeImportSession(sessionId);
},
};
@@ -0,0 +1 @@
export { importHandlers } from './handlers';
@@ -3,10 +3,14 @@ import type { MessagePort } from 'node:worker_threads';
import type { EventBasedChannel } from 'async-call-rpc';
import { AsyncCall } from 'async-call-rpc';
import { ipcRenderer } from 'electron';
import { ipcRenderer, webUtils } from 'electron';
import { Subject } from 'rxjs';
import { z } from 'zod';
import type {
CreateImportSessionFromSourceOptions,
NativeImportBrowserSource,
} from '../shared/import';
import {
AFFINE_API_CHANNEL_NAME,
AFFINE_EVENT_CHANNEL_NAME,
@@ -275,9 +279,61 @@ function getHelperAPIs() {
const mainAPIs = getMainAPIs();
const helperAPIs = getHelperAPIs();
type DirectoryImportFile = File & { webkitRelativePath?: string };
function filePathFromFile(file: File) {
return webUtils.getPathForFile(file);
}
function directoryPathFromFiles(files: File[]) {
const first = files.find(
(file): file is DirectoryImportFile =>
!!(file as DirectoryImportFile).webkitRelativePath
);
if (!first) return null;
const filePath = filePathFromFile(first);
if (!filePath) return null;
const relativePath = first.webkitRelativePath;
if (!relativePath) return null;
const relativeParts = relativePath.split('/');
let rootPath = filePath.replaceAll('\\', '/');
for (let i = relativeParts.length - 1; i > 0; i--) {
const part = relativeParts[i];
if (part && rootPath.endsWith(`/${part}`)) {
rootPath = rootPath.slice(0, -part.length - 1);
}
}
return rootPath || null;
}
function resolveNativeImportSource(source: NativeImportBrowserSource) {
if (source.kind === 'file') {
const path = filePathFromFile(source.file);
return path ? { kind: 'filePath', path } : null;
}
const path = directoryPathFromFiles(source.files);
return path ? { kind: 'directoryPath', path } : null;
}
export const apis = {
...mainAPIs.apis,
...helperAPIs.apis,
import: {
...mainAPIs.apis.import,
createImportSessionFromSource(
options: CreateImportSessionFromSourceOptions
) {
const source = resolveNativeImportSource(options.source);
if (!source) {
throw new Error('Native import requires a local file source');
}
return mainAPIs.apis.import.createImportSession({
format: options.format,
source,
batchLimits: options.batchLimits,
});
},
},
};
export const events = {
@@ -0,0 +1,28 @@
export type NativeImportFormat =
| 'markdownZip'
| 'notionZip'
| 'obsidian'
| 'bearZip';
export type NativeImportBrowserSource =
| { kind: 'file'; file: File }
| { kind: 'directory'; files: File[] };
export type CreateImportSessionFromSourceOptions = {
format: NativeImportFormat;
source: NativeImportBrowserSource;
batchLimits?: {
maxDocs?: number;
maxBlobs?: number;
maxBlobBytes?: number;
};
};
export type NativeImportSessionHandlers = {
createImportSession(
options: CreateImportSessionFromSourceOptions
): Promise<string> | string;
nextImportBatch(sessionId: string): Promise<string | null> | string | null;
cancelImportSession(sessionId: string): Promise<void> | void;
disposeImportSession(sessionId: string): Promise<void> | void;
};
@@ -0,0 +1,547 @@
import 'fake-indexeddb/auto';
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
import { getAFFiNEWorkspaceSchema } from '@affine/core/modules/workspace';
import type { DocSnapshot } from '@blocksuite/affine/store';
import { TestWorkspace } from '@blocksuite/affine/store/test';
import type { ImportBatch } from '@blocksuite/affine/widgets/linked-doc';
import { describe, expect, test, vi } from 'vitest';
import { ImportCommitService } from './commit-service';
function docSnapshot(id: string, title: string): DocSnapshot {
return {
type: 'page',
meta: {
id,
title,
createDate: 0,
tags: [],
},
blocks: {
type: 'block',
id: `block:${id}`,
flavour: 'affine:page',
props: {
title: {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
},
},
children: [
{
type: 'block',
id: `block:${id}:note`,
flavour: 'affine:note',
props: {},
children: [
{
type: 'block',
id: `block:${id}:paragraph`,
flavour: 'affine:paragraph',
props: {
type: 'text',
text: {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
},
},
children: [],
},
],
},
],
},
};
}
function createFolderTree() {
const folders = new Map<string, FolderNode>();
const links: { parentId: string; docId: string }[] = [];
let nextId = 0;
class FolderNode {
readonly children: string[] = [];
constructor(readonly id: string) {
folders.set(id, this);
}
createFolder() {
const id = `folder-${++nextId}`;
this.children.push(id);
new FolderNode(id);
return id;
}
createLink(...[, docId]: ['doc', string]) {
links.push({ parentId: this.id, docId });
}
indexAt() {
return 'after';
}
}
const rootFolder = new FolderNode('root');
return {
links,
service: {
folderTree: {
rootFolder,
['folderNode$']: (id: string) => ({ value: folders.get(id) }),
},
},
};
}
function createCommitService(
collection: TestWorkspace,
options: {
organizeService?: unknown;
explorerIconService?: unknown;
tagService?: unknown;
} = {}
) {
return new ImportCommitService({
collection,
schema: getAFFiNEWorkspaceSchema(),
extensions: getStoreManager().config.init().value.get('store'),
organizeService: options.organizeService as never,
explorerIconService: options.explorerIconService as never,
tagService: options.tagService as never,
logger: {
warn: vi.fn(),
},
});
}
describe('ImportCommitService', () => {
test('commits native batch blobs, docs, folders, icons, and warnings', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const folderTree = createFolderTree();
const setIcon = vi.fn();
const service = createCommitService(collection, {
organizeService: folderTree.service,
explorerIconService: { setIcon },
});
const batch: ImportBatch = {
blobs: [
{
blobId: 'blob-1',
sourcePath: 'assets/image.png',
fileName: 'image.png',
mime: 'image/png',
bytes: new Uint8Array([1, 2, 3]),
},
],
docs: [
{
id: 'doc-1',
snapshot: docSnapshot('doc-1', 'Imported'),
meta: { title: 'Committed title', favorite: true },
},
],
folders: [
{ path: 'root-folder', name: 'Root folder' },
{
path: 'root-folder/doc-1',
name: 'Imported',
parentPath: 'root-folder',
pageId: 'doc-1',
icon: { type: 'emoji', unicode: '✅' },
},
],
warnings: [{ code: 'lossy', message: 'Dropped unsupported block' }],
done: true,
};
const result = await service.commitBatch(batch);
expect(result).toEqual({
docIds: ['doc-1'],
rootFolderId: 'folder-1',
warnings: batch.warnings,
});
await expect(collection.blobSync.get('blob-1')).resolves.toBeInstanceOf(
File
);
expect(collection.getDoc('doc-1')).not.toBeNull();
expect(collection.meta.getDocMeta('doc-1')).toMatchObject({
title: 'Committed title',
favorite: true,
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
expect(setIcon).toHaveBeenCalledWith({
where: 'doc',
id: 'doc-1',
icon: { type: 'emoji', unicode: '✅' },
});
});
test('records doc commit failures as warnings and continues remaining docs', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const service = createCommitService(collection);
const result = await service.commitBatch({
blobs: [],
docs: [
{
id: 'doc-skip',
sourcePath: 'docs/skip.md',
snapshot: { ...docSnapshot('doc-skip', 'Skip'), blocks: null },
} as never,
{
id: 'doc-ok',
sourcePath: 'docs/ok.md',
snapshot: docSnapshot('doc-ok', 'Ok'),
meta: { title: 'Ok' },
},
],
done: true,
});
expect(result.docIds).toEqual(['doc-ok']);
expect(result.warnings).toEqual([
{
code: 'skipped_doc',
sourcePath: 'docs/skip.md',
message:
'Skipped docs/skip.md: document snapshot could not be committed',
},
]);
expect(collection.getDoc('doc-ok')).not.toBeNull();
});
test('records doc meta failures as warnings without dropping committed docs', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const service = createCommitService(collection);
const originalSetDocMeta = collection.meta.setDocMeta.bind(collection.meta);
const setDocMeta = vi.spyOn(collection.meta, 'setDocMeta');
setDocMeta.mockImplementation((id, meta) => {
if (id === 'doc-meta') {
throw new Error('meta failed');
}
return originalSetDocMeta(id, meta);
});
try {
const result = await service.commitBatch({
blobs: [],
docs: [
{
id: 'doc-meta',
sourcePath: 'docs/meta.md',
snapshot: docSnapshot('doc-meta', 'Meta'),
meta: { title: 'Meta' },
},
{
id: 'doc-ok',
sourcePath: 'docs/ok.md',
snapshot: docSnapshot('doc-ok', 'Ok'),
meta: { title: 'Ok' },
},
],
done: true,
});
expect(result.docIds).toEqual(['doc-meta', 'doc-ok']);
expect(result.warnings).toEqual([
{
code: 'doc_meta_failed',
sourcePath: 'docs/meta.md',
message: 'Failed to apply metadata for docs/meta.md: meta failed',
},
]);
expect(collection.getDoc('doc-meta')).not.toBeNull();
expect(collection.getDoc('doc-ok')).not.toBeNull();
expect(collection.meta.getDocMeta('doc-ok')).toMatchObject({
title: 'Ok',
});
} finally {
setDocMeta.mockRestore();
}
});
test('commits native tag names as workspace tags', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const tags = new Map<string, { id: string; value: string }>();
const service = createCommitService(collection, {
tagService: {
randomTagColor: () => 'red',
tagList: {
['tags$']: {
value: [],
},
createTag: (value: string) => {
const tag = { id: `tag-${tags.size + 1}`, value };
tags.set(value, tag);
return tag;
},
},
},
});
await service.commitBatch({
blobs: [],
docs: [
{
id: 'doc-1',
snapshot: docSnapshot('doc-1', 'Tagged'),
meta: { tags: ['Blue Tag'], title: 'Tagged' },
},
],
tags: [{ name: 'work/project', docIds: ['doc-1'] }],
done: true,
});
expect([...tags.values()]).toEqual([
{ id: 'tag-1', value: 'Blue Tag' },
{ id: 'tag-2', value: 'work' },
]);
expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual([
'tag-1',
'tag-2',
]);
});
test('commits batch folders, icons, and collapsed Bear root tags', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
collection.createDoc('doc-2');
const folderTree = createFolderTree();
const setIcon = vi.fn();
const tags = new Map<string, { id: string; value: string }>();
const service = createCommitService(collection, {
organizeService: folderTree.service,
explorerIconService: { setIcon },
tagService: {
randomTagColor: () => 'red',
tagList: {
['tags$']: {
value: [],
},
createTag: (value: string) => {
const tag = { id: `tag-${tags.size + 1}`, value };
tags.set(value, tag);
return tag;
},
},
},
});
const result = await service.commitBatch({
docs: [],
blobs: [],
icons: [{ docId: 'doc-1', icon: { type: 'emoji', unicode: '📘' } }],
tags: [
{ name: 'work/project', docIds: ['doc-1'] },
{ name: 'work/research', docIds: ['doc-2'] },
],
folders: [
{ path: 'Bear', name: 'Bear' },
{
path: 'Bear/Idea',
name: 'Idea',
parentPath: 'Bear',
pageId: 'doc-1',
},
],
done: true,
});
expect(result).toEqual({
docIds: [],
rootFolderId: 'folder-1',
warnings: [],
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
expect(setIcon).toHaveBeenCalledWith({
where: 'doc',
id: 'doc-1',
icon: { type: 'emoji', unicode: '📘' },
});
expect([...tags.values()]).toEqual([{ id: 'tag-1', value: 'work' }]);
expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual(['tag-1']);
expect(collection.meta.getDocMeta('doc-2')?.tags).toEqual(['tag-1']);
});
test('keeps folder and doc links idempotent across repeated commits', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
const batch: ImportBatch = {
docs: [],
blobs: [],
folders: [
{ path: 'Root', name: 'Root' },
{
path: 'Root/Doc',
name: 'Doc',
parentPath: 'Root',
pageId: 'doc-1',
},
],
done: true,
};
const first = await service.commitBatch(batch);
const second = await service.commitBatch(batch);
expect(first.rootFolderId).toBe('folder-1');
expect(second.rootFolderId).toBe('folder-1');
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
});
test('resolves partial batch folder links when parent arrives later', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [
{
path: 'Root/Doc',
name: 'Doc',
parentPath: 'Root',
pageId: 'doc-1',
},
],
done: false,
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [{ path: 'Root', name: 'Root' }],
done: true,
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
});
test('warns and clears unresolved folders on final batch', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
const result = await service.commitBatch({
docs: [],
blobs: [],
folders: [
{
path: 'Missing/Doc',
name: 'Doc',
parentPath: 'Missing',
pageId: 'doc-1',
},
],
done: true,
});
expect(folderTree.links).toEqual([]);
expect(result.warnings).toEqual([
{
code: 'unresolved_folder',
sourcePath: 'Missing/Doc',
message:
'Skipped folder placement for Missing/Doc: parent folder was not found',
},
]);
});
test('applies icons for root-level imported pages', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const setIcon = vi.fn();
const service = createCommitService(collection, {
organizeService: folderTree.service,
explorerIconService: { setIcon },
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [
{
path: 'Doc',
name: 'Doc',
pageId: 'doc-1',
icon: { type: 'emoji', unicode: '📌' },
},
],
done: true,
});
expect(setIcon).toHaveBeenCalledWith({
where: 'doc',
id: 'doc-1',
icon: { type: 'emoji', unicode: '📌' },
});
});
test('keeps Notion page nodes usable as child containers', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('project');
collection.createDoc('nested');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [
{ path: 'Export', name: 'Export' },
{
path: 'Export/Project',
name: 'Project',
parentPath: 'Export',
pageId: 'project',
},
{
path: 'Export/Project/Nested',
name: 'Nested',
parentPath: 'Export/Project',
pageId: 'nested',
},
],
done: true,
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'project' },
{ parentId: 'folder-2', docId: 'nested' },
]);
});
});
@@ -0,0 +1,333 @@
import type { IconData } from '@affine/component';
import type { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon';
import type { OrganizeService } from '@affine/core/modules/organize';
import type { TagService } from '@affine/core/modules/tag';
import {
type ExtensionType,
type Schema,
Transformer,
type Workspace,
} from '@blocksuite/affine/store';
import type {
ImportBatch,
ImportCommitResult,
ImportFolder,
ImportIconData,
} from '@blocksuite/affine/widgets/linked-doc';
type Logger = {
warn: (message: string, ...args: unknown[]) => void;
};
type CommitServiceOptions = {
collection: Workspace;
schema: Schema;
extensions: ExtensionType[];
organizeService?: OrganizeService;
explorerIconService?: ExplorerIconService;
tagService?: TagService;
logger: Logger;
};
type EmojiIconData = Extract<IconData, { unicode: string }>;
const EMOJI_ICON_TYPE = 'emoji' as EmojiIconData['type'];
function copyToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
function errorMessage(error: unknown) {
return error instanceof Error
? error.message || error.name
: 'Unknown error occurred';
}
export class ImportCommitService {
private readonly folderIdByPath = new Map<string, string>();
private readonly linkedDocsByFolder = new Set<string>();
private readonly pendingFolders: ImportFolder[] = [];
constructor(private readonly options: CommitServiceOptions) {}
async commitBatch(batch: ImportBatch): Promise<ImportCommitResult> {
const warnings = [...(batch.warnings ?? [])];
for (const blob of batch.blobs) {
const bytes = new Uint8Array(blob.bytes);
await this.options.collection.blobSync.set(
blob.blobId,
new File([copyToArrayBuffer(bytes)], blob.fileName, {
type: blob.mime,
})
);
}
const transformer = new Transformer({
schema: this.options.schema,
blobCRUD: this.options.collection.blobSync,
docCRUD: {
create: (id: string) =>
this.options.collection.createDoc(id).getStore({ id }),
get: (id: string) =>
this.options.collection.getDoc(id)?.getStore({ id }) ?? null,
delete: (id: string) => this.options.collection.removeDoc(id),
},
middlewares: [],
});
const docIds: string[] = [];
const tags = new Map<string, string[]>();
for (const doc of batch.docs) {
let store: Awaited<ReturnType<Transformer['snapshotToDoc']>>;
try {
store = await transformer.snapshotToDoc(doc.snapshot);
} catch (error) {
warnings.push({
code: 'skipped_doc',
sourcePath: doc.sourcePath,
message: `Skipped ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`,
});
continue;
}
if (!store) {
warnings.push({
code: 'skipped_doc',
sourcePath: doc.sourcePath,
message: `Skipped ${doc.sourcePath ?? doc.id}: document snapshot could not be committed`,
});
continue;
}
docIds.push(store.id);
if (doc.meta && Object.keys(doc.meta).length) {
try {
const { tags: docTags, ...meta } = doc.meta;
if (Object.keys(meta).length) {
this.options.collection.meta.setDocMeta(store.id, meta);
}
for (const tag of docTags ?? []) {
const docIds = tags.get(tag) ?? [];
docIds.push(store.id);
tags.set(tag, docIds);
}
} catch (error) {
warnings.push({
code: 'doc_meta_failed',
sourcePath: doc.sourcePath,
message: `Failed to apply metadata for ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`,
});
}
}
}
for (const tag of batch.tags ?? []) {
const docIds = tags.get(tag.name) ?? [];
docIds.push(...tag.docIds);
tags.set(tag.name, docIds);
}
const rootFolderId = this.applyNativeFolders(
batch.folders ?? [],
warnings,
batch.done
);
this.applyNativeTags(tags);
this.applyNativeIcons(batch.icons);
return {
docIds,
entryId: batch.entryId,
isWorkspaceFile: batch.isWorkspaceFile,
rootFolderId,
warnings,
};
}
private applyNativeFolders(
folders: ImportFolder[],
warnings: ImportCommitResult['warnings'],
batchDone: boolean
): string | undefined {
const { organizeService } = this.options;
if (folders.length === 0) return undefined;
if (!organizeService) {
for (const folder of folders) {
if (folder.pageId && !folder.parentPath) {
this.applyIcon(folder.pageId, folder.icon);
}
}
return undefined;
}
try {
let rootFolderId: string | undefined;
this.pendingFolders.push(...folders);
let progressed = true;
while (progressed && this.pendingFolders.length) {
progressed = false;
const nextPending: ImportFolder[] = [];
const pending = this.pendingFolders.splice(0);
const childParentPaths = new Set(
pending
.map(folder => folder.parentPath)
.filter((path): path is string => !!path)
);
for (const folder of pending) {
const needsContainer =
!folder.pageId || childParentPaths.has(folder.path);
if (needsContainer && !this.folderIdByPath.has(folder.path)) {
const parent = folder.parentPath
? this.folderIdByPath.has(folder.parentPath)
? organizeService.folderTree.folderNode$(
this.folderIdByPath.get(folder.parentPath) ?? ''
).value
: null
: organizeService.folderTree.rootFolder;
if (!parent) {
nextPending.push(folder);
continue;
}
const folderId = parent.createFolder(
folder.name,
parent.indexAt('after')
);
this.folderIdByPath.set(folder.path, folderId);
rootFolderId ??= folderId;
progressed = true;
} else if (needsContainer) {
rootFolderId ??= this.folderIdByPath.get(folder.path);
}
if (folder.pageId) {
if (!this.applyFolderDocLink(folder)) {
nextPending.push(folder);
continue;
}
progressed = true;
}
}
if (progressed && nextPending.length) {
this.pendingFolders.push(...nextPending);
} else if (batchDone) {
for (const folder of nextPending) {
warnings.push({
code: 'unresolved_folder',
sourcePath: folder.path,
message: `Skipped folder placement for ${folder.path}: parent folder was not found`,
});
}
break;
} else {
this.pendingFolders.push(...nextPending);
break;
}
}
return rootFolderId;
} catch (error) {
this.options.logger.warn('Failed to commit import folders:', error);
return undefined;
}
}
private applyFolderDocLink(folder: ImportFolder): boolean {
const { organizeService } = this.options;
if (!folder.pageId) return true;
if (!folder.parentPath) {
this.applyIcon(folder.pageId, folder.icon);
return true;
}
if (!organizeService) return true;
const parentFolderId = this.folderIdByPath.get(folder.parentPath);
if (!parentFolderId) return false;
const linkKey = `${parentFolderId}:${folder.pageId}`;
if (!this.linkedDocsByFolder.has(linkKey)) {
const parent =
organizeService.folderTree.folderNode$(parentFolderId).value;
if (!parent) return false;
parent.createLink('doc', folder.pageId, parent.indexAt('after'));
this.linkedDocsByFolder.add(linkKey);
}
this.applyIcon(folder.pageId, folder.icon);
return true;
}
private applyNativeIcons(icons?: ImportBatch['icons']) {
for (const icon of icons ?? []) {
this.applyIcon(icon.docId, icon.icon);
}
}
private applyIcon(id: string, icon?: ImportIconData) {
if (!icon || !this.options.explorerIconService) return;
const iconData = toIconData(icon);
if (!iconData) return;
this.options.explorerIconService.setIcon({
where: 'doc',
id,
icon: iconData,
});
}
private applyNativeTags(tags?: Map<string, string[]>) {
const { tagService, collection } = this.options;
if (!tagService || !tags?.size) return;
try {
const existingTagMap = new Map<string, string>();
for (const tag of tagService.tagList.tags$.value) {
existingTagMap.set(tag.value$.value.toLowerCase(), tag.id);
}
const rootTagDocMap = new Map<
string,
{ displayName: string; docs: Set<string> }
>();
for (const [tagName, tagDocIds] of tags) {
const originalRoot = tagName.split('/')[0];
const key = originalRoot.toLowerCase();
let entry = rootTagDocMap.get(key);
if (!entry) {
entry = { displayName: originalRoot, docs: new Set() };
rootTagDocMap.set(key, entry);
}
for (const docId of tagDocIds) {
entry.docs.add(docId);
}
}
for (const [rootTagKey, { displayName, docs }] of rootTagDocMap) {
let tagId = existingTagMap.get(rootTagKey);
if (!tagId) {
const newTag = tagService.tagList.createTag(
displayName,
tagService.randomTagColor()
);
tagId = newTag.id;
existingTagMap.set(rootTagKey, tagId);
}
for (const docId of docs) {
const doc = collection.getDoc(docId);
const currentTags = doc?.meta?.tags ?? [];
if (!currentTags.includes(tagId)) {
collection.meta.setDocMeta(docId, {
tags: [...currentTags, tagId],
});
}
}
}
} catch (error) {
this.options.logger.warn('Failed to commit import tags:', error);
}
}
}
function toIconData(icon: ImportIconData): IconData | undefined {
if (icon.type === 'emoji') {
return {
type: EMOJI_ICON_TYPE,
unicode: icon.unicode,
};
}
return undefined;
}
@@ -1,10 +1,4 @@
import {
Button,
IconButton,
type IconData,
IconType,
Modal,
} from '@affine/component';
import { Button, IconButton, Modal } from '@affine/component';
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks';
import { useNavigateHelper } from '@affine/core/components/hooks/use-navigate-helper';
@@ -13,9 +7,10 @@ import {
GlobalDialogService,
type WORKSPACE_DIALOG_SCHEMA,
} from '@affine/core/modules/dialogs';
import { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon';
import { OrganizeService } from '@affine/core/modules/organize';
import { TagService } from '@affine/core/modules/tag';
import {
type ImportRunContext,
ImportService,
} from '@affine/core/modules/import';
import { UrlService } from '@affine/core/modules/url';
import {
getAFFiNEWorkspaceSchema,
@@ -28,12 +23,10 @@ import track from '@affine/track';
import { openDirectory, openFilesWith } from '@blocksuite/affine/shared/utils';
import type { Workspace } from '@blocksuite/affine/store';
import {
BearTransformer,
DocxTransformer,
HtmlTransformer,
type ImportWarning,
MarkdownTransformer,
NotionHtmlTransformer,
ObsidianTransformer,
ZipTransformer,
} from '@blocksuite/affine/widgets/linked-doc';
import {
@@ -54,6 +47,7 @@ import {
type SVGAttributes,
useCallback,
useMemo,
useRef,
useState,
} from 'react';
@@ -61,177 +55,15 @@ import * as style from './styles.css';
const logger = new DebugLogger('import');
type NotionPageIcon = {
type: 'emoji' | 'image';
content: string; // emoji unicode or image URL/data
};
type FolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
pageId?: string;
parentPath?: string;
icon?: NotionPageIcon;
};
// Helper function to create folder structure using OrganizeService
function createFolderStructure(
organizeService: OrganizeService,
hierarchy: FolderHierarchy,
parentFolderId: string | null = null,
explorerIconService?: ExplorerIconService
): {
folderId: string | null;
docLinks: Array<{ folderId: string; docId: string }>;
} {
const docLinks: Array<{ folderId: string; docId: string }> = [];
const rootFolder = organizeService.folderTree.rootFolder;
function processHierarchyNode(
node: FolderHierarchy,
currentParentId: string | null
): string | null {
let currentFolderId = currentParentId;
// If this node represents a folder (has children but no pageId), create it
if (node.children.size > 0 && !node.pageId && node.name) {
const parent = currentParentId
? organizeService.folderTree.folderNode$(currentParentId).value
: rootFolder;
if (parent) {
const index = parent.indexAt('after');
currentFolderId = parent.createFolder(node.name, index);
}
}
// Process all children
for (const child of node.children.values()) {
if (child.pageId) {
// This is a document, link it to the current folder
if (currentFolderId) {
docLinks.push({ folderId: currentFolderId, docId: child.pageId });
}
// Set icon for the document if available
if (child.icon && explorerIconService) {
logger.debug('=== Setting icon for document ===');
logger.debug('Document ID:', child.pageId);
logger.debug('Icon data:', child.icon);
try {
let iconData: IconData | undefined;
if (child.icon.type === 'emoji') {
iconData = {
type: IconType.Emoji,
unicode: child.icon.content,
};
logger.debug('Created emoji icon data:', iconData);
} else if (child.icon.type === 'image') {
// For image icons, we'd need to handle blob conversion
// For now, let's skip image icons or convert them to default
// This could be enhanced later to download and convert images to blobs
logger.debug(
'Skipping image icon (not implemented):',
child.icon.content
);
iconData = undefined;
}
if (iconData) {
logger.debug('Calling explorerIconService.setIcon with:', {
where: 'doc',
id: child.pageId,
icon: iconData,
});
explorerIconService.setIcon({
where: 'doc',
id: child.pageId,
icon: iconData,
});
logger.debug('Icon set successfully for document:', child.pageId);
} else {
logger.debug('No valid icon data to set');
}
} catch (error) {
logger.error(
'Error setting icon for document:',
child.pageId,
error
);
logger.warn(
'Failed to set icon for document:',
child.pageId,
error
);
}
} else {
if (!child.icon) {
logger.debug('No icon found for document:', child.pageId);
}
if (!explorerIconService) {
logger.debug(
'ExplorerIconService not available for document:',
child.pageId
);
}
}
} else if (child.children.size > 0) {
// This is a subfolder, process it recursively
processHierarchyNode(child, currentFolderId);
}
}
return currentFolderId;
}
const rootFolderId = processHierarchyNode(hierarchy, parentFolderId);
return { folderId: rootFolderId, docLinks };
}
/**
* Creates the folder tree described by {@link folderHierarchy} via
* {@link OrganizeService} and links every document into its folder.
* Returns the root folder ID on success, or `undefined` if the
* hierarchy is empty or an error occurs.
*
* When {@link explorerIconService} is provided, document icons from the
* hierarchy (e.g. Notion page emojis) are applied. Callers that do not
* need icon support can omit it safely.
*/
function applyFolderHierarchy(
organizeService: OrganizeService,
folderHierarchy: FolderHierarchy,
explorerIconService?: ExplorerIconService
): string | undefined {
if (folderHierarchy.children.size === 0) return undefined;
try {
const { folderId, docLinks } = createFolderStructure(
organizeService,
folderHierarchy,
null,
explorerIconService
);
for (const { folderId, docId } of docLinks) {
const folder = organizeService.folderTree.folderNode$(folderId).value;
if (folder) {
const index = folder.indexAt('after');
folder.createLink('doc', docId, index);
}
}
return folderId || undefined;
} catch (error) {
logger.warn('Failed to create folder structure:', error);
return undefined;
}
function shouldSnapshotPickedFiles(type: ImportType, acceptType: AcceptType) {
if (acceptType === 'Directory' || acceptType === 'Skip') return false;
return !['markdownZip', 'notion', 'bear'].includes(type);
}
type ImportType =
| 'markdown'
| 'markdownZip'
| 'notion'
| 'notionMarkdown'
| 'obsidian'
| 'bear'
| 'snapshot'
@@ -240,28 +72,71 @@ type ImportType =
| 'dotaffinefile';
type AcceptType = 'Markdown' | 'Zip' | 'Html' | 'Docx' | 'Directory' | 'Skip'; // Skip is used for dotaffinefile
type Status = 'idle' | 'importing' | 'success' | 'error';
type ImportErrorState = {
code: string;
message: string;
sourcePath?: string;
};
type ImportResult = {
docIds: string[];
entryId?: string;
isWorkspaceFile?: boolean;
rootFolderId?: string;
importedWorkspace?: WorkspaceMetadata;
warnings?: ImportWarning[];
};
type ImportedWorkspacePayload = {
workspace: WorkspaceMetadata;
};
type ImportFunctionArgs = {
docCollection: Workspace;
files: File[];
importAffineFile: () => Promise<WorkspaceMetadata | undefined>;
importService?: ImportService;
context: ImportRunContext;
};
function toImportErrorState(error: unknown): ImportErrorState {
const sourcePath =
typeof error === 'object' &&
error !== null &&
'sourcePath' in error &&
typeof error.sourcePath === 'string'
? error.sourcePath
: undefined;
if (error instanceof DOMException && error.name === 'AbortError') {
return {
code: 'cancelled',
message: 'Import cancelled',
sourcePath,
};
}
if (error instanceof Error) {
return {
code: error.name || 'import-error',
message: error.message || 'Unknown error occurred',
sourcePath,
};
}
return {
code: 'unknown',
message: 'Unknown error occurred',
sourcePath,
};
}
function requireImportService(importService?: ImportService) {
if (!importService) {
throw new Error('Import service is unavailable');
}
return importService;
}
type ImportConfig = {
fileOptions: { acceptType: AcceptType; multiple: boolean };
importFunction: (
docCollection: Workspace,
files: File[],
handleImportAffineFile: () => Promise<WorkspaceMetadata | undefined>,
organizeService?: OrganizeService,
explorerIconService?: ExplorerIconService,
tagService?: TagService
) => Promise<ImportResult>;
importFunction: (args: ImportFunctionArgs) => Promise<ImportResult>;
};
const importOptions = [
@@ -319,17 +194,6 @@ const importOptions = [
testId: 'editor-option-menu-import-notion',
type: 'notion' as ImportType,
},
{
key: 'notionMarkdown',
label: 'com.affine.import.notion-markdown',
prefixIcon: <NotionIcon color={cssVar('black')} width={20} height={20} />,
suffixIcon: (
<HelpIcon color={cssVarV2('icon/primary')} width={20} height={20} />
),
suffixTooltip: 'com.affine.import.notion-markdown.tooltip',
testId: 'editor-option-menu-import-notion-markdown',
type: 'notionMarkdown' as ImportType,
},
{
key: 'obsidian',
label: 'com.affine.import.obsidian',
@@ -400,13 +264,7 @@ const importOptions = [
const importConfigs: Record<ImportType, ImportConfig> = {
markdown: {
fileOptions: { acceptType: 'Markdown', multiple: true },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
importFunction: async ({ docCollection, files }) => {
const docIds: string[] = [];
for (const file of files) {
const text = await file.text();
@@ -427,45 +285,20 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
markdownZip: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
_explorerIconService
) => {
importFunction: async ({ files, importService, context }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for markdownZip import');
}
const { docIds, folderHierarchy } =
await MarkdownTransformer.importMarkdownZip({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(organizeService, folderHierarchy)
: undefined;
return {
docIds,
rootFolderId,
};
return requireImportService(importService).importMarkdownZip(
file,
context
);
},
},
html: {
fileOptions: { acceptType: 'Html', multiple: true },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
importFunction: async ({ docCollection, files }) => {
const docIds: string[] = [];
for (const file of files) {
const text = await file.text();
@@ -486,220 +319,39 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
notion: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
explorerIconService
) => {
importFunction: async ({ files, importService, context }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for notion import');
}
const { entryId, pageIds, isWorkspaceFile, folderHierarchy } =
await NotionHtmlTransformer.importNotionZip({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(
organizeService,
folderHierarchy,
explorerIconService
)
: undefined;
return {
docIds: pageIds,
entryId,
isWorkspaceFile,
rootFolderId,
};
},
},
notionMarkdown: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
explorerIconService
) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for notionMarkdown import');
}
const { docIds, folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(
organizeService,
folderHierarchy,
explorerIconService
)
: undefined;
return {
docIds,
rootFolderId,
};
return requireImportService(importService).importNotionZip(file, context);
},
},
obsidian: {
fileOptions: { acceptType: 'Directory', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
explorerIconService
) => {
const { docIds, docEmojis } =
await ObsidianTransformer.importObsidianVault({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
importedFiles: files,
extensions: getStoreManager().config.init().value.get('store'),
});
if (explorerIconService) {
for (const [id, emoji] of docEmojis.entries()) {
explorerIconService.setIcon({
where: 'doc',
id,
icon: { type: IconType.Emoji, unicode: emoji },
});
}
}
return { docIds };
importFunction: async ({ files, importService, context }) => {
return requireImportService(importService).importObsidianVault(
files,
context
);
},
},
bear: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
_explorerIconService,
tagService
) => {
importFunction: async ({ files, importService, context }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single .bear2bk file for Bear import');
}
let docIds: string[];
let tags: Map<string, string[]>;
let folderHierarchy: FolderHierarchy;
try {
const result = await BearTransformer.importBearBackup({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
docIds = result.docIds;
tags = result.tags;
folderHierarchy = result.folderHierarchy;
} catch (err) {
logger.error('Bear import failed:', err);
throw err instanceof Error
? err
: new Error(String(err) || 'Bear import failed');
}
// Create AFFiNE tags from Bear tags
if (tagService && tags.size > 0) {
try {
// Get existing tags for deduplication
const existingTags = tagService.tagList.tags$.value;
const existingTagMap = new Map<string, string>(); // lowercase name → tag id
for (const tag of existingTags) {
const name = tag.value$.value.toLowerCase();
existingTagMap.set(name, tag.id);
}
// Consolidate tags by root segment (e.g., "privat/bike" → "privat").
// Keyed by lowercase root for case-insensitive dedup, but the
// original capitalization of the first occurrence is preserved
// so new AFFiNE tags are created with the user's casing.
const rootTagDocMap = new Map<
string,
{ displayName: string; docs: Set<string> }
>();
for (const [tagName, tagDocIds] of tags) {
const originalRoot = tagName.split('/')[0];
const key = originalRoot.toLowerCase();
let entry = rootTagDocMap.get(key);
if (!entry) {
entry = { displayName: originalRoot, docs: new Set<string>() };
rootTagDocMap.set(key, entry);
}
for (const docId of tagDocIds) {
entry.docs.add(docId);
}
}
for (const [
rootTagKey,
{ displayName, docs: docIdSet },
] of rootTagDocMap) {
// Check if tag already exists (case-insensitive)
let tagId = existingTagMap.get(rootTagKey);
if (!tagId) {
const newTag = tagService.tagList.createTag(
displayName,
tagService.randomTagColor()
);
tagId = newTag.id;
existingTagMap.set(rootTagKey, tagId);
}
// Assign tag to each doc
for (const docId of docIdSet) {
const doc = docCollection.getDoc(docId);
const currentTags = doc?.meta?.tags ?? [];
if (!currentTags.includes(tagId)) {
docCollection.meta.setDocMeta(docId, {
tags: [...currentTags, tagId],
});
}
}
}
} catch (error) {
logger.warn('Failed to create Bear tags:', error);
}
}
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(organizeService, folderHierarchy)
: undefined;
return {
docIds,
rootFolderId,
};
return requireImportService(importService).importBearBackup(
file,
context
);
},
},
docx: {
fileOptions: { acceptType: 'Docx', multiple: false },
importFunction: async (docCollection, file) => {
const files = Array.isArray(file) ? file : [file];
importFunction: async ({ docCollection, files }) => {
const docIds: string[] = [];
for (const file of files) {
const docId = await DocxTransformer.importDocx({
@@ -715,13 +367,7 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
snapshot: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
importFunction: async ({ docCollection, files }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for snapshot import');
@@ -743,14 +389,8 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
dotaffinefile: {
fileOptions: { acceptType: 'Skip', multiple: false },
importFunction: async (
_,
__,
handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
const workspace = await handleImportAffineFile();
importFunction: async ({ importAffineFile }) => {
const workspace = await importAffineFile();
return {
docIds: [],
entryId: undefined,
@@ -843,8 +483,18 @@ const ImportOptions = ({
);
};
const ImportingStatus = () => {
const ImportingStatus = ({
progress,
onCancel,
}: {
progress: { completed: number; total: number } | null;
onCancel: () => void;
}) => {
const t = useI18n();
const progressLabel =
progress && progress.total > 0
? `${progress.completed}/${progress.total}`
: null;
return (
<>
<div className={style.importModalTitle}>
@@ -853,11 +503,25 @@ const ImportingStatus = () => {
<p className={style.importStatusContent}>
{t['com.affine.import.status.importing.message']()}
</p>
{progressLabel ? (
<div className={style.importProgress}>{progressLabel}</div>
) : null}
<div className={style.importModalButtonContainer}>
<Button onClick={onCancel} variant="secondary">
{t['Cancel']()}
</Button>
</div>
</>
);
};
const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => {
const SuccessStatus = ({
warnings,
onComplete,
}: {
warnings: string[];
onComplete: () => void;
}) => {
const t = useI18n();
return (
<>
@@ -876,6 +540,13 @@ const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => {
</a>
.
</p>
{warnings.length ? (
<div className={style.importWarnings}>
{warnings.map((warning, index) => (
<div key={`${warning}-${index}`}>{warning}</div>
))}
</div>
) : null}
<div className={style.importModalButtonContainer}>
<Button onClick={onComplete} variant="primary">
{t['Complete']()}
@@ -889,7 +560,7 @@ const ErrorStatus = ({
error,
onRetry,
}: {
error: string | null;
error: ImportErrorState | null;
onRetry: () => void;
}) => {
const t = useI18n();
@@ -900,8 +571,11 @@ const ErrorStatus = ({
{t['com.affine.import.status.failed.title']()}
</div>
<p className={style.importStatusContent}>
{error || 'Unknown error occurred'}
{error?.message || 'Unknown error occurred'}
</p>
{error?.sourcePath ? (
<div className={style.importErrorDetail}>{error.sourcePath}</div>
) : null}
<div className={style.importModalButtonContainer}>
<Button
onClick={() => {
@@ -924,13 +598,16 @@ export const ImportDialog = ({
}: DialogComponentProps<WORKSPACE_DIALOG_SCHEMA['import']>) => {
const t = useI18n();
const [status, setStatus] = useState<Status>('idle');
const [importError, setImportError] = useState<string | null>(null);
const [importError, setImportError] = useState<ImportErrorState | null>(null);
const [importResult, setImportResult] = useState<ImportResult | null>(null);
const [importProgress, setImportProgress] = useState<{
completed: number;
total: number;
} | null>(null);
const importAbortControllerRef = useRef<AbortController | null>(null);
const workspace = useService(WorkspaceService).workspace;
const docCollection = workspace.docCollection;
const organizeService = useService(OrganizeService);
const explorerIconService = useService(ExplorerIconService);
const tagService = useService(TagService);
const importService = useService(ImportService);
const globalDialogService = useService(GlobalDialogService);
@@ -984,6 +661,7 @@ export const ImportDialog = ({
const handleImport = useAsyncCallback(
async (type: ImportType) => {
setImportError(null);
setImportProgress(null);
try {
const importConfig = importConfigs[type];
const { acceptType, multiple } = importConfig.fileOptions;
@@ -992,8 +670,13 @@ export const ImportDialog = ({
acceptType === 'Skip'
? []
: acceptType === 'Directory'
? await openDirectory()
: await openFilesWith(acceptType, multiple);
? await openDirectory({
fileSystemAccess: false,
})
: await openFilesWith(acceptType, multiple, {
fileSystemAccess: false,
snapshot: shouldSnapshotPickedFiles(type, acceptType),
});
if (!files || (files.length === 0 && acceptType !== 'Skip')) {
throw new Error(
@@ -1009,20 +692,28 @@ export const ImportDialog = ({
});
}
const abortController = new AbortController();
importAbortControllerRef.current = abortController;
const {
docIds,
entryId,
isWorkspaceFile,
rootFolderId,
importedWorkspace,
} = await importConfig.importFunction(
warnings,
} = await importConfig.importFunction({
docCollection,
files,
handleImportAffineFile,
organizeService,
explorerIconService,
tagService
);
importAffineFile: handleImportAffineFile,
importService,
context: {
signal: abortController.signal,
onProgress: progress => {
setImportProgress(progress);
},
},
});
importAbortControllerRef.current = null;
setImportResult({
docIds,
@@ -1030,6 +721,7 @@ export const ImportDialog = ({
isWorkspaceFile,
rootFolderId,
importedWorkspace,
warnings,
});
setStatus('success');
track.$.importModal.$.import({
@@ -1043,26 +735,19 @@ export const ImportDialog = ({
control: 'import',
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : 'Unknown error occurred';
setImportError(errorMessage);
importAbortControllerRef.current = null;
const structuredError = toImportErrorState(error);
setImportError(structuredError);
setStatus('error');
track.$.importModal.$.import({
type,
status: 'failed',
error: errorMessage || undefined,
error: structuredError.message || undefined,
});
logger.error('Failed to import', error);
}
},
[
docCollection,
explorerIconService,
handleImportAffineFile,
organizeService,
tagService,
t,
]
[docCollection, handleImportAffineFile, importService, t]
);
const finishImport = useCallback(() => {
@@ -1073,8 +758,11 @@ export const ImportDialog = ({
close();
return;
}
const { importedWorkspace: _workspace, ...result } = importResult;
close(result);
close({
docIds: importResult.docIds,
entryId: importResult.entryId,
isWorkspaceFile: importResult.isWorkspaceFile,
});
}, [close, handleCreatedWorkspace, importResult]);
const handleComplete = useCallback(() => {
@@ -1082,13 +770,27 @@ export const ImportDialog = ({
}, [finishImport]);
const handleRetry = () => {
setImportProgress(null);
setStatus('idle');
};
const handleCancel = useCallback(() => {
importAbortControllerRef.current?.abort();
}, []);
const statusComponents = {
idle: <ImportOptions onImport={handleImport} />,
importing: <ImportingStatus />,
success: <SuccessStatus onComplete={handleComplete} />,
importing: (
<ImportingStatus progress={importProgress} onCancel={handleCancel} />
),
success: (
<SuccessStatus
warnings={(importResult?.warnings ?? []).map(warning =>
typeof warning === 'string' ? warning : warning.message
)}
onComplete={handleComplete}
/>
),
error: <ErrorStatus error={importError} onRetry={handleRetry} />,
};
@@ -0,0 +1,111 @@
import {
getNativeImportSessionHandlers,
type NativeImportSessionHandlers,
} from '@affine/core/modules/import/runtime-config';
import type {
NativeImportBrowserSource,
NativeImportFormat,
} from '@affine/electron-api';
import type {
ImportBatch,
ImportCommitResult,
} from '@blocksuite/affine/widgets/linked-doc';
import type { ImportCommitService } from './commit-service';
const NATIVE_IMPORT_BATCH_LIMITS = {
maxDocs: 20,
maxBlobs: 20,
maxBlobBytes: 10 * 1024 * 1024,
};
export async function commitNativeImport(
format: NativeImportFormat,
source: File | File[],
commitService: ImportCommitService,
options: {
signal?: AbortSignal;
onProgress?: (progress: { completed: number; total: number }) => void;
} = {}
): Promise<ImportCommitResult> {
const native = await getNativeImporter();
if (options.signal?.aborted) {
throw new DOMException('Import cancelled', 'AbortError');
}
const sessionId = await native.createImportSession({
format,
source: getNativeImportSource(source),
batchLimits: NATIVE_IMPORT_BATCH_LIMITS,
});
const docIds: string[] = [];
const warnings: ImportCommitResult['warnings'] = [];
let entryId: string | undefined;
let isWorkspaceFile = false;
let rootFolderId: string | undefined;
let cancelled = false;
try {
for (;;) {
if (options.signal?.aborted) {
cancelled = true;
try {
await native.cancelImportSession(sessionId);
} catch {
// Preserve the AbortError reported to callers.
}
throw new DOMException('Import cancelled', 'AbortError');
}
const payload = await native.nextImportBatch(sessionId);
if (!payload) break;
const batch = JSON.parse(payload) as ImportBatch;
options.onProgress?.(batch.progress ?? { completed: 0, total: 0 });
const result = await commitService.commitBatch(batch);
docIds.push(...result.docIds);
warnings.push(...result.warnings);
entryId ??= batch.entryId;
isWorkspaceFile ||= !!batch.isWorkspaceFile;
rootFolderId ??= result.rootFolderId;
}
} catch (error) {
if (!cancelled) {
try {
await native.cancelImportSession(sessionId);
} catch {
// Preserve the original import error.
}
}
throw error;
} finally {
await native.disposeImportSession(sessionId);
}
if (!docIds.length && !isWorkspaceFile) {
throw new Error('No importable documents were found in the selected file.');
}
return {
docIds,
entryId,
isWorkspaceFile,
rootFolderId,
warnings,
};
}
async function getNativeImporter(): Promise<NativeImportSessionHandlers> {
const registered = getNativeImportSessionHandlers();
if (registered) {
return registered;
}
throw new Error('Native import session handlers are not registered');
}
function getNativeImportSource(
source: File | File[]
): NativeImportBrowserSource {
if (Array.isArray(source)) {
return { kind: 'directory', files: source };
}
return { kind: 'file', file: source };
}
@@ -57,6 +57,30 @@ export const importStatusContent = style({
color: cssVar('textPrimaryColor'),
});
export const importProgress = style({
width: '100%',
fontSize: cssVar('fontSm'),
lineHeight: cssVar('lineHeight'),
color: cssVar('textSecondaryColor'),
});
export const importWarnings = style({
width: '100%',
maxHeight: '120px',
overflow: 'auto',
fontSize: cssVar('fontSm'),
lineHeight: cssVar('lineHeight'),
color: cssVar('textSecondaryColor'),
});
export const importErrorDetail = style({
width: '100%',
fontSize: cssVar('fontXs'),
lineHeight: cssVar('lineHeight'),
color: cssVar('textSecondaryColor'),
overflowWrap: 'anywhere',
});
export const importModalButtonContainer = style({
width: '100%',
display: 'flex',
@@ -0,0 +1,106 @@
import { describe, expect, test } from 'vitest';
import {
preflightWebFilesImport,
preflightWebZipImport,
WebImportLimitError,
} from './web-limits';
function zipDirectoryFile(names: string[]) {
const chunks = names.map(name => {
const nameBytes = new TextEncoder().encode(name);
const header = new Uint8Array(46 + nameBytes.length);
writeU32(header, 0, 0x02014b50);
writeU32(header, 20, 1);
writeU32(header, 24, 1);
writeU16(header, 28, nameBytes.length);
header.set(nameBytes, 46);
return header;
});
return new File(chunks, 'import.zip', { type: 'application/zip' });
}
function writeU16(bytes: Uint8Array, offset: number, value: number) {
bytes[offset] = value & 0xff;
bytes[offset + 1] = (value >> 8) & 0xff;
}
function writeU32(bytes: Uint8Array, offset: number, value: number) {
bytes[offset] = value & 0xff;
bytes[offset + 1] = (value >> 8) & 0xff;
bytes[offset + 2] = (value >> 16) & 0xff;
bytes[offset + 3] = (value >> 24) & 0xff;
}
describe('preflightWebZipImport', () => {
test('rejects nested zip before JS importer parsing', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['entry.md', 'nested.zip']))
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('rejects document count over web limit', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['a.md', 'b.md']), {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('counts html pages as web import documents', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['a.html', 'b.htm']), {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('allows small zip directory', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['entry.md']), {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).resolves.toBeUndefined();
});
});
describe('preflightWebFilesImport', () => {
test('rejects directory import over web document limit', async () => {
await expect(
preflightWebFilesImport(
[new File(['a'], 'a.md'), new File(['b'], 'b.md')],
{
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
}
)
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('allows small directory import', async () => {
await expect(
preflightWebFilesImport([new File(['a'], 'a.md')], {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).resolves.toBeUndefined();
});
});
@@ -0,0 +1,155 @@
export type WebImportLimits = {
maxTotalBytes: number;
maxEntryBytes: number;
maxEntryCount: number;
maxNestedZipDepth: number;
maxDocumentCount: number;
};
export const webImportLimits: WebImportLimits = {
maxTotalBytes: 32 * 1024 * 1024,
maxEntryBytes: 8 * 1024 * 1024,
maxEntryCount: 1000,
maxNestedZipDepth: 0,
maxDocumentCount: 250,
};
export class WebImportLimitError extends Error {
constructor(message: string) {
super(message);
this.name = 'WebImportLimitError';
}
}
export async function preflightWebZipImport(
file: File,
limits = webImportLimits
) {
if (file.size > limits.maxTotalBytes) {
throw new WebImportLimitError(
'This import is too large for the web app. Please import it in the desktop client.'
);
}
const entries = readZipDirectory(new Uint8Array(await file.arrayBuffer()));
if (entries.length > limits.maxEntryCount) {
throw new WebImportLimitError(
'This import has too many files for the web app. Please import it in the desktop client.'
);
}
const documentCount = entries.filter(entry =>
isWebImportDocument(entry.name)
).length;
if (documentCount > limits.maxDocumentCount) {
throw new WebImportLimitError(
'This import has too many documents for the web app. Please import it in the desktop client.'
);
}
for (const entry of entries) {
if (entry.uncompressedSize > limits.maxEntryBytes) {
throw new WebImportLimitError(
'This import contains a file that is too large for the web app. Please import it in the desktop client.'
);
}
if (
limits.maxNestedZipDepth === 0 &&
entry.name.toLowerCase().endsWith('.zip')
) {
throw new WebImportLimitError(
'This import contains a nested zip. Please import it in the desktop client.'
);
}
}
}
export async function preflightWebFilesImport(
files: File[],
limits = webImportLimits
) {
const totalBytes = files.reduce((total, file) => total + file.size, 0);
if (totalBytes > limits.maxTotalBytes) {
throw new WebImportLimitError(
'This import is too large for the web app. Please import it in the desktop client.'
);
}
if (files.length > limits.maxEntryCount) {
throw new WebImportLimitError(
'This import has too many files for the web app. Please import it in the desktop client.'
);
}
let documentCount = 0;
for (const file of files) {
if (file.size > limits.maxEntryBytes) {
throw new WebImportLimitError(
'This import contains a file that is too large for the web app. Please import it in the desktop client.'
);
}
const path = file.webkitRelativePath || file.name;
if (path.toLowerCase().endsWith('.zip')) {
throw new WebImportLimitError(
'This import contains a nested zip. Please import it in the desktop client.'
);
}
if (isWebImportDocument(path)) {
documentCount += 1;
}
}
if (documentCount > limits.maxDocumentCount) {
throw new WebImportLimitError(
'This import has too many documents for the web app. Please import it in the desktop client.'
);
}
}
function isWebImportDocument(path: string) {
const lower = path.toLowerCase();
return (
lower.endsWith('.md') || lower.endsWith('.html') || lower.endsWith('.htm')
);
}
type ZipEntry = {
name: string;
uncompressedSize: number;
};
function readZipDirectory(bytes: Uint8Array): ZipEntry[] {
const entries: ZipEntry[] = [];
for (let offset = 0; offset + 46 <= bytes.length; offset++) {
if (readU32(bytes, offset) !== 0x02014b50) continue;
const compressedSize = readU32(bytes, offset + 20);
const uncompressedSize = readU32(bytes, offset + 24);
const fileNameLength = readU16(bytes, offset + 28);
const extraLength = readU16(bytes, offset + 30);
const commentLength = readU16(bytes, offset + 32);
const nameStart = offset + 46;
const nameEnd = nameStart + fileNameLength;
if (nameEnd > bytes.length) break;
entries.push({
name: new TextDecoder().decode(bytes.subarray(nameStart, nameEnd)),
uncompressedSize: uncompressedSize || compressedSize,
});
offset = nameEnd + extraLength + commentLength - 1;
}
return entries;
}
function readU16(bytes: Uint8Array, offset: number) {
return bytes[offset] | (bytes[offset + 1] << 8);
}
function readU32(bytes: Uint8Array, offset: number) {
return (
(bytes[offset] |
(bytes[offset + 1] << 8) |
(bytes[offset + 2] << 16) |
(bytes[offset + 3] << 24)) >>>
0
);
}
@@ -0,0 +1,30 @@
import type { Framework } from '@toeverything/infra';
import { ExplorerIconService } from '../explorer-icon/services/explorer-icon';
import { OrganizeService } from '../organize';
import { TagService } from '../tag';
import { WorkspaceScope, WorkspaceService } from '../workspace';
import { ImportService } from './services/service';
export { ImportService };
export type { NativeImportSessionHandlers } from './runtime-config';
export {
getNativeImportSessionHandlers,
registerNativeImportSessionHandlers,
} from './runtime-config';
export type { ImportRunContext } from './services/service';
export type {
NativeImportBrowserSource,
NativeImportFormat,
} from '@affine/electron-api';
export function configureImportModule(framework: Framework) {
framework
.scope(WorkspaceScope)
.service(ImportService, [
WorkspaceService,
OrganizeService,
ExplorerIconService,
TagService,
]);
}
@@ -0,0 +1,19 @@
import type { NativeImportSessionHandlers } from '@affine/electron-api';
export type {
NativeImportBrowserSource,
NativeImportFormat,
NativeImportSessionHandlers,
} from '@affine/electron-api';
let nativeImportSessionHandlers: NativeImportSessionHandlers | null = null;
export function registerNativeImportSessionHandlers(
handlers: NativeImportSessionHandlers | null
) {
nativeImportSessionHandlers = handlers;
}
export function getNativeImportSessionHandlers() {
return nativeImportSessionHandlers;
}
@@ -0,0 +1,190 @@
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
import { ImportCommitService } from '@affine/core/desktop/dialogs/import/commit-service';
import { commitNativeImport } from '@affine/core/desktop/dialogs/import/native-backend';
import {
preflightWebFilesImport,
preflightWebZipImport,
} from '@affine/core/desktop/dialogs/import/web-limits';
import { DebugLogger } from '@affine/debug';
import { snapshotFile } from '@blocksuite/affine/shared/utils';
import {
BearTransformer,
type ImportWarning,
MarkdownTransformer,
NotionHtmlTransformer,
ObsidianTransformer,
Unzip,
} from '@blocksuite/affine/widgets/linked-doc';
import { Service } from '@toeverything/infra';
import type { ExplorerIconService } from '../../explorer-icon/services/explorer-icon';
import type { OrganizeService } from '../../organize';
import type { TagService } from '../../tag';
import type { WorkspaceService } from '../../workspace';
import { getAFFiNEWorkspaceSchema } from '../../workspace';
const logger = new DebugLogger('import');
export type ImportRunContext = {
signal?: AbortSignal;
onProgress?: (progress: { completed: number; total: number }) => void;
};
export class ImportService extends Service {
constructor(
private readonly workspaceService: WorkspaceService,
private readonly organizeService: OrganizeService,
private readonly explorerIconService: ExplorerIconService,
private readonly tagService: TagService
) {
super();
}
async importMarkdownZip(file: File, context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({ organize: true });
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('markdownZip', file, commitService, context);
}
await preflightWebZipImport(file);
const snapshot = await snapshotFile(file);
const { batch } = await MarkdownTransformer.planMarkdownZip({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
async importNotionZip(file: File, context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({
organize: true,
explorerIcon: true,
});
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('notionZip', file, commitService, context);
}
await preflightWebZipImport(file);
const snapshot = await snapshotFile(file);
const format = await detectNotionZipFormat(snapshot);
if (format === 'markdown') {
const { batch } = await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
const { batch } = await NotionHtmlTransformer.planNotionHtmlZip({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
async importObsidianVault(files: File[], context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({
explorerIcon: true,
});
if (!BUILD_CONFIG.isElectron) {
await preflightWebFilesImport(files);
}
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('obsidian', files, commitService, context);
}
const { files: snapshots, warnings } = await snapshotReadableFiles(files);
if (!snapshots.length) {
throw new Error('No readable files were found in the selected folder.');
}
const { batch } = await ObsidianTransformer.planObsidianVault({
collection,
schema: getAFFiNEWorkspaceSchema(),
importedFiles: snapshots,
extensions: getStoreManager().config.init().value.get('store'),
});
batch.warnings = [...(batch.warnings ?? []), ...warnings];
return commitService.commitBatch(batch);
}
async importBearBackup(file: File, context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({
organize: true,
tag: true,
});
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('bearZip', file, commitService, context);
}
await preflightWebZipImport(file);
const snapshot = await snapshotFile(file);
const { batch } = await BearTransformer.planBearBackup({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
private createCommitService(options: {
organize?: boolean;
explorerIcon?: boolean;
tag?: boolean;
}) {
return new ImportCommitService({
collection: this.workspaceService.workspace.docCollection,
schema: getAFFiNEWorkspaceSchema(),
extensions: getStoreManager().config.init().value.get('store'),
organizeService: options.organize ? this.organizeService : undefined,
explorerIconService: options.explorerIcon
? this.explorerIconService
: undefined,
tagService: options.tag ? this.tagService : undefined,
logger,
});
}
}
async function detectNotionZipFormat(file: File): Promise<'markdown' | 'html'> {
const unzip = new Unzip();
await unzip.load(file);
let hasHtml = false;
for (const entry of unzip) {
const lower = entry.path.toLowerCase();
if (lower.endsWith('.md')) return 'markdown';
if (lower.endsWith('.html') && !lower.endsWith('/index.html')) {
hasHtml = true;
}
}
if (hasHtml) return 'html';
throw new Error('No Notion Markdown or HTML pages found in the archive');
}
async function snapshotReadableFiles(files: File[]) {
const snapshots: File[] = [];
const warnings: ImportWarning[] = [];
for (const file of files) {
try {
snapshots.push(await snapshotFile(file));
} catch (error) {
const sourcePath = file.webkitRelativePath || file.name;
const reason = error instanceof Error ? error.message : String(error);
warnings.push({
code: 'file-unreadable',
message: `Skipped unreadable file: ${sourcePath}. ${reason}`,
sourcePath,
});
}
}
return { files: snapshots, warnings };
}
@@ -34,6 +34,7 @@ import { configureFeatureFlagModule } from './feature-flag';
import { configureGlobalContextModule } from './global-context';
import { configureI18nModule } from './i18n';
import { configureIconPickerModule } from './icon-picker';
import { configureImportModule } from './import';
import { configureImportClipperModule } from './import-clipper';
import { configureImportTemplateModule } from './import-template';
import { configureIntegrationModule } from './integration';
@@ -103,6 +104,7 @@ export function configureCommonModules(framework: Framework) {
configureSystemFontFamilyModule(framework);
configureEditorSettingModule(framework);
configureImportTemplateModule(framework);
configureImportModule(framework);
configureUserspaceModule(framework);
configureAppSidebarModule(framework);
configureJournalModule(framework);
+19 -3
View File
@@ -8,12 +8,18 @@ import type {
} from '@affine/electron/main/exposed';
import type { AppInfo } from '@affine/electron/preload/electron-api';
import type { SharedStorage } from '@affine/electron/preload/shared-storage';
import type { CreateImportSessionFromSourceOptions } from '@affine/electron/shared/import';
type MainHandlers = typeof mainHandlers;
type HelperHandlers = typeof helperHandlers;
type HelperEvents = typeof helperEvents;
type MainEvents = typeof mainEvents;
export type ClientHandler = {
type ImportPreloadHandlers = {
createImportSessionFromSource(
options: CreateImportSessionFromSourceOptions
): Promise<string>;
};
type MainClientHandlers = {
[namespace in keyof MainHandlers]: {
[method in keyof MainHandlers[namespace]]: MainHandlers[namespace][method] extends (
arg0: any,
@@ -26,7 +32,12 @@ export type ClientHandler = {
: Promise<ReturnType<MainHandlers[namespace][method]>>
: never;
};
} & HelperHandlers;
};
export type ClientHandler = Omit<MainClientHandlers, 'import'> &
HelperHandlers & {
import: MainClientHandlers['import'] & ImportPreloadHandlers;
};
export type ClientEvents = MainEvents & HelperEvents;
export const appInfo = (globalThis as any).__appInfo as AppInfo | null;
@@ -38,7 +49,6 @@ export const sharedStorage = (globalThis as any).__sharedStorage as
| undefined;
export type { AppInfo, SharedStorage };
export {
type SpellCheckStateSchema,
type TabViewsMetaSchema,
@@ -51,3 +61,9 @@ export type {
AddTabOption,
TabAction,
} from '@affine/electron/main/windows-manager';
export type {
CreateImportSessionFromSourceOptions,
NativeImportBrowserSource,
NativeImportFormat,
NativeImportSessionHandlers,
} from '@affine/electron/shared/import';
@@ -3,25 +3,25 @@
"ca": 92,
"da": 4,
"de": 100,
"el-GR": 91,
"el-GR": 90,
"en": 100,
"es-AR": 91,
"es-CL": 92,
"es": 91,
"fa": 91,
"fa": 90,
"fr": 95,
"hi": 1,
"it": 92,
"ja": 91,
"ja": 90,
"kk": 98,
"ko": 91,
"nb-NO": 45,
"pl": 92,
"pt-BR": 91,
"pt-BR": 90,
"ru": 93,
"sv-SE": 91,
"tr": 98,
"uk": 91,
"uk": 90,
"ur": 98,
"zh-Hans": 100,
"zh-Hant": 93
@@ -1,2 +0,0 @@
recordings
.env
@@ -1,38 +0,0 @@
{
"name": "@affine/media-capture-playground",
"private": true,
"type": "module",
"version": "0.27.0",
"scripts": {
"dev:web": "affine bundle --dev",
"build:web": "affine bundle",
"dev:server": "node --env-file-if-exists=.env --watch server/main.ts"
},
"dependencies": {
"@affine/native": "workspace:*",
"@google/generative-ai": "^0.24.0",
"@types/express": "^5.0.0",
"@types/lodash-es": "^4.17.12",
"@types/multer": "^2.0.0",
"@types/react": "^19.0.8",
"@types/react-dom": "^19.0.3",
"chokidar": "^4.0.3",
"express": "^5.0.0",
"express-rate-limit": "^7.1.5",
"fs-extra": "^11.3.0",
"lodash-es": "^4.17.23",
"multer": "^2.2.0",
"react": "^19.2.1",
"react-dom": "^19.2.1",
"react-markdown": "^10.1.0",
"socket.io": "^4.8.3",
"socket.io-client": "^4.8.3",
"swr": "^2.3.7",
"tailwindcss": "^4.1.17"
},
"devDependencies": {
"@types/fs-extra": "^11",
"@types/react": "^19.0.1",
"@types/react-dom": "^19.0.2"
}
}
@@ -1,58 +0,0 @@
export function createWavBuffer(
samples: Float32Array,
options: {
sampleRate?: number;
numChannels?: number;
}
) {
const { sampleRate = 44100, numChannels = 1 } = options;
const bitsPerSample = 16;
const bytesPerSample = bitsPerSample / 8;
const dataSize = samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataSize); // WAV header is 44 bytes
const view = new DataView(buffer);
// Write WAV header
// "RIFF" chunk descriptor
writeString(view, 0, 'RIFF');
view.setUint32(4, 36 + dataSize, true); // File size - 8
writeString(view, 8, 'WAVE');
// "fmt " sub-chunk
writeString(view, 12, 'fmt ');
view.setUint32(16, 16, true); // Sub-chunk size
view.setUint16(20, 1, true); // Audio format (1 = PCM)
view.setUint16(22, numChannels, true); // Channels
view.setUint32(24, sampleRate, true); // Sample rate
view.setUint32(28, sampleRate * numChannels * bytesPerSample, true); // Byte rate
view.setUint16(32, numChannels * bytesPerSample, true); // Block align
view.setUint16(34, bitsPerSample, true); // Bits per sample
// "data" sub-chunk
writeString(view, 36, 'data');
view.setUint32(40, dataSize, true); // Sub-chunk size
// Write audio data
const offset = 44;
for (let i = 0; i < samples.length; i++) {
// Convert float32 to int16
const s = Math.max(-1, Math.min(1, samples[i]));
view.setInt16(
offset + i * bytesPerSample,
s < 0 ? s * 0x8000 : s * 0x7fff,
true
);
}
return buffer;
}
function writeString(
view: DataView<ArrayBuffer>,
offset: number,
string: string
) {
for (let i = 0; i < string.length; i++) {
view.setUint8(offset + i, string.charCodeAt(i));
}
}
@@ -1,200 +0,0 @@
import { GoogleGenerativeAI } from '@google/generative-ai';
import {
GoogleAIFileManager,
type UploadFileResponse,
} from '@google/generative-ai/server';
const DEFAULT_MODEL = 'gemini-2.5-pro';
export interface TranscriptionResult {
title: string;
summary: string;
segments: {
speaker: string;
start_time: string;
end_time: string;
transcription: string;
}[];
}
const PROMPT_TRANSCRIPTION = `
Generate audio transcription and diarization for the recording.
The recording source is most likely from a video call with multiple speakers.
Output in JSON format with the following structure:
{
"segments": [
{
"speaker": "Speaker A",
"start_time": "MM:SS",
"end_time": "MM:SS",
"transcription": "..."
},
...
],
}
- Use consistent speaker labels throughout
- Accurate timestamps in MM:SS format
- Clean transcription with proper punctuation
- Identify speakers by name if possible, otherwise use "Speaker A/B/C"
`;
const PROMPT_SUMMARY = `
Generate a short title and summary of the conversation. The input is in the following JSON format:
{
"segments": [
{
"speaker": "Speaker A",
"start_time": "MM:SS",
"end_time": "MM:SS",
"transcription": "..."
},
...
],
}
Output in JSON format with the following structure:
{
"title": "Title of the recording",
"summary": "Summary of the conversation in markdown format"
}
1. Summary Structure:
- The sumary should be inferred from the speakers' language and context
- All insights should be derived directly from speakers' language and context
- Use hierarchical organization for clear information structure
- Use markdown format for the summary. Use bullet points, lists and other markdown styles when appropriate
2. Title:
- Come up with a title for the recording.
- The title should be a short description of the recording.
- The title should be a single sentence or a few words.
`;
export async function gemini(
audioFilePath: string,
options?: {
model?: 'gemini-2.5-flash' | 'gemini-2.5-pro';
mode?: 'transcript' | 'summary';
}
) {
if (!process.env.GOOGLE_GEMINI_API_KEY) {
console.error('Missing GOOGLE_GEMINI_API_KEY environment variable');
throw new Error('GOOGLE_GEMINI_API_KEY is not set');
}
// Initialize GoogleGenerativeAI and FileManager with your API_KEY
const genAI = new GoogleGenerativeAI(process.env.GOOGLE_GEMINI_API_KEY);
const fileManager = new GoogleAIFileManager(
process.env.GOOGLE_GEMINI_API_KEY
);
async function transcribe(
audioFilePath: string
): Promise<TranscriptionResult | null> {
let uploadResult: UploadFileResponse | null = null;
try {
// Upload the audio file
uploadResult = await fileManager.uploadFile(audioFilePath, {
mimeType: 'audio/wav',
displayName: 'audio_transcription.wav',
});
console.log('File uploaded:', uploadResult.file.uri);
// Initialize a Gemini model appropriate for your use case.
const model = genAI.getGenerativeModel({
model: options?.model || DEFAULT_MODEL,
generationConfig: {
responseMimeType: 'application/json',
},
});
// Generate content using a prompt and the uploaded file
const result = await model.generateContent([
{
fileData: {
fileUri: uploadResult.file.uri,
mimeType: uploadResult.file.mimeType,
},
},
{
text: PROMPT_TRANSCRIPTION,
},
]);
const text = result.response.text();
try {
const parsed = JSON.parse(text);
return parsed;
} catch (e) {
console.error('Failed to parse transcription JSON:', e);
console.error('Raw text that failed to parse:', text);
return null;
}
} catch (e) {
console.error('Error during transcription:', e);
return null;
} finally {
if (uploadResult) {
await fileManager.deleteFile(uploadResult.file.name);
}
}
}
async function summarize(transcription: TranscriptionResult) {
try {
const model = genAI.getGenerativeModel({
model: options?.model || DEFAULT_MODEL,
generationConfig: {
responseMimeType: 'application/json',
},
});
const result = await model.generateContent([
{
text: PROMPT_SUMMARY + '\n\n' + JSON.stringify(transcription),
},
]);
const text = result.response.text();
try {
const parsed = JSON.parse(text);
return parsed;
} catch (e) {
console.error('Failed to parse summary JSON:', e);
console.error('Raw text that failed to parse:', text);
return null;
}
} catch (e) {
console.error('Error during summarization:', e);
return null;
}
}
const transcription = await transcribe(audioFilePath);
if (!transcription) {
console.error('Transcription failed');
return null;
}
const summary = await summarize(transcription);
if (!summary) {
console.error('Summary generation failed');
return transcription;
}
const result = {
...transcription,
...summary,
};
console.log('Processing completed:', {
title: result.title,
segmentsCount: result.segments?.length,
});
return result;
}
File diff suppressed because it is too large Load Diff
@@ -1,4 +0,0 @@
declare module '*.txt' {
const content: string;
export default content;
}
@@ -1,7 +0,0 @@
{
"extends": "../../../tsconfig.node.json",
"compilerOptions": {
"rootDir": "./server"
},
"include": ["./server"]
}
@@ -1,10 +0,0 @@
{
"extends": "../../../tsconfig.web.json",
"compilerOptions": {
"rootDir": "./web",
"outDir": "./dist",
"tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo"
},
"include": ["./web", "server/types.d.ts"],
"references": [{ "path": "../native" }]
}
@@ -1,38 +0,0 @@
import { AppList } from './components/app-list';
import { GlobalRecordButton } from './components/global-record-button';
import { SavedRecordings } from './components/saved-recordings';
export function App() {
return (
<div className="h-screen bg-gray-50 overflow-hidden">
<div className="h-full p-4 flex gap-4 max-w-[1800px] mx-auto">
<div className="flex-1 flex flex-col min-h-0">
<div className="flex justify-between items-center mb-1">
<h1 className="text-xl font-bold text-gray-900">
Running Applications
</h1>
<GlobalRecordButton />
</div>
<p className="text-sm text-gray-500 mb-2">
Select an application to start recording its audio, or use global
recording for system-wide audio
</p>
<div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 overflow-auto">
<AppList />
</div>
</div>
<div className="w-[1024px] flex flex-col min-h-0">
<h1 className="text-xl font-bold text-gray-900 mb-1">
Saved Recordings
</h1>
<p className="text-sm text-gray-500 mb-2">
Listen to and manage your recorded audio files
</p>
<div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 p-4 overflow-auto">
<SavedRecordings />
</div>
</div>
</div>
</div>
);
}
@@ -1,122 +0,0 @@
import React from 'react';
import type { AppGroup, RecordingStatus } from '../types';
import { formatDuration } from '../utils';
interface AppItemProps {
app: AppGroup;
recordings?: RecordingStatus[];
}
export function AppItem({ app, recordings }: AppItemProps) {
const [imgError, setImgError] = React.useState(false);
const [isRecording, setIsRecording] = React.useState(false);
const appName = app.rootApp.name || '';
const bundleId = app.rootApp.bundleIdentifier || '';
const firstLetter = appName.charAt(0).toUpperCase();
const isRunning = app.apps.some(a => a.isRunning);
const recording = recordings?.find((r: RecordingStatus) =>
app.apps.some(a => a.processId === r.processId)
);
const handleRecordClick = React.useCallback(() => {
const recordingApp = app.apps.find(a => a.isRunning);
if (!recordingApp) {
return;
}
if (isRecording) {
void fetch(`/api/apps/${recordingApp.processId}/stop`, {
method: 'POST',
})
.then(() => setIsRecording(false))
.catch(error => console.error('Failed to stop recording:', error));
} else {
void fetch(`/api/apps/${recordingApp.processId}/record`, {
method: 'POST',
})
.then(() => setIsRecording(true))
.catch(error => console.error('Failed to start recording:', error));
}
}, [app.apps, isRecording]);
React.useEffect(() => {
setIsRecording(!!recording);
}, [recording]);
const [duration, setDuration] = React.useState(0);
React.useEffect(() => {
if (recording) {
const interval = setInterval(() => {
setDuration(Date.now() - recording.startTime);
}, 1000);
return () => clearInterval(interval);
} else {
setDuration(0);
}
return () => {};
}, [recording]);
return (
<div className="flex items-center h-16 space-x-2 p-3 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100">
{imgError ? (
<div className="w-8 h-8 rounded-lg bg-gray-50 border border-gray-100 flex items-center justify-center text-gray-600 font-semibold text-base">
{firstLetter}
</div>
) : (
<img
src={`/api/apps/${app.rootApp.processId}/icon`}
loading="lazy"
alt={appName}
className="w-8 h-8 object-contain rounded-lg bg-gray-50 border border-gray-100"
onError={() => setImgError(true)}
/>
)}
<div className="flex-1 min-w-0">
<div className="flex items-center space-x-1 mb-1">
{appName ? (
<span className="text-gray-900 font-medium text-sm truncate">
{appName}
</span>
) : (
<span className="text-gray-400 italic font-medium text-sm">
Unnamed Application
</span>
)}
<span className="text-xs px-1 bg-gray-50 text-gray-500 rounded border border-gray-100">
PID: {app.rootApp.processId}
</span>
<span
className={`text-xs px-2 py-0.5 rounded-full font-medium border ${recording ? 'bg-red-50 text-red-600 border-red-100 opacity-100' : 'opacity-0'}`}
>
{recording ? formatDuration(duration) : '00:00:00'}
</span>
</div>
<div className="text-xs text-gray-500 font-mono truncate opacity-80">
{bundleId}
</div>
</div>
{(isRunning || isRecording) && (
<button
onClick={handleRecordClick}
className={`h-8 min-w-[80px] flex items-center justify-center rounded-lg text-sm font-medium transition-all duration-200 ${
isRecording
? 'bg-red-50 text-red-600 hover:bg-red-100 border border-red-200'
: 'bg-blue-50 text-blue-600 hover:bg-blue-100 border border-blue-200'
}`}
>
{isRecording ? (
<>
<div className="w-1.5 h-1.5 rounded-full bg-red-500 animate-pulse mr-2" />
<span>Stop</span>
</>
) : (
<span>Record</span>
)}
</button>
)}
</div>
);
}
@@ -1,145 +0,0 @@
import React from 'react';
import useSWRSubscription from 'swr/subscription';
import type { App, AppGroup, RecordingStatus } from '../types';
import { socket } from '../utils';
import { AppItem } from './app-item';
export function AppList() {
const { data: apps = [] } = useSWRSubscription('apps', (_key, { next }) => {
let apps: App[] = [];
// Initial apps fetch
fetch('/api/apps')
.then(res => res.json())
.then(data => {
apps = data.apps;
next(null, apps);
})
.catch(err => next(err));
// Subscribe to app updates
socket.on('apps:all', data => {
next(null, data.apps);
apps = data.apps;
});
socket.on('apps:state-changed', data => {
const index = apps.findIndex(a => a.processId === data.processId);
console.log('apps:state-changed', data, index);
if (index !== -1) {
next(
null,
apps.toSpliced(index, 1, {
...apps[index],
isRunning: data.isRunning,
})
);
}
});
socket.on('connect', () => {
// Refetch on reconnect
fetch('/api/apps')
.then(res => res.json())
.then(data => next(null, data.apps))
.catch(err => next(err));
});
return () => {
socket.off('apps:all');
socket.off('apps:state-changed');
socket.off('connect');
};
});
const { data: recordings = [] } = useSWRSubscription<RecordingStatus[]>(
'recordings',
(
_key: string,
{ next }: { next: (err: Error | null, data?: RecordingStatus[]) => void }
) => {
// Subscribe to recording updates
socket.on('apps:recording', (data: { recordings: RecordingStatus[] }) => {
next(null, data.recordings);
});
return () => {
socket.off('apps:recording');
};
}
);
const appGroups: AppGroup[] = React.useMemo(() => {
const mapping = apps.reduce((acc: Record<number, AppGroup>, app: App) => {
if (!acc[app.processGroupId]) {
acc[app.processGroupId] = {
processGroupId: app.processGroupId,
apps: [],
rootApp:
apps.find((a: App) => a.processId === app.processGroupId) || app,
};
}
acc[app.processGroupId].apps.push(app);
return acc;
}, {});
return Object.values(mapping);
}, [apps]);
const runningApps = (appGroups || []).filter(app =>
app.apps.some(a => a.isRunning)
);
const notRunningApps = (appGroups || []).filter(
app => !app.apps.some(a => a.isRunning)
);
return (
<div className="h-full flex flex-col divide-y divide-gray-100">
<div className="p-4 relative">
<div className="flex items-center justify-between sticky top-0 bg-white z-10 mb-2">
<h2 className="text-sm font-semibold text-gray-900">
Active Applications
</h2>
<span className="text-xs px-2 py-1 bg-blue-50 rounded-full text-blue-600 font-medium">
{runningApps.length} listening
</span>
</div>
<div className="space-y-2">
{runningApps.map(app => (
<AppItem
key={app.processGroupId}
app={app}
recordings={recordings}
/>
))}
{runningApps.length === 0 && (
<div className="text-sm text-gray-500 italic bg-gray-50 rounded-xl p-4 text-center">
No applications are currently listening
</div>
)}
</div>
</div>
<div className="p-4 flex-1 relative">
<div className="flex items-center justify-between sticky top-0 bg-white z-10 mb-2">
<h2 className="text-sm font-semibold text-gray-900">
Other Applications
</h2>
<span className="text-xs px-2 py-1 bg-gray-50 rounded-full text-gray-600 font-medium">
{notRunningApps.length} available
</span>
</div>
<div className="space-y-2">
{notRunningApps.map(app => (
<AppItem
key={app.processGroupId}
app={app}
recordings={recordings}
/>
))}
{notRunningApps.length === 0 && (
<div className="text-sm text-gray-500 italic bg-gray-50 rounded-xl p-4 text-center">
No other applications found
</div>
)}
</div>
</div>
</div>
);
}
@@ -1,70 +0,0 @@
import { useCallback, useEffect, useState } from 'react';
import { socket } from '../utils';
export function GlobalRecordButton() {
const [isRecording, setIsRecording] = useState(false);
const [isLoading, setIsLoading] = useState(false);
useEffect(() => {
function handleRecordingStatus(data: {
recordings: Array<{ processId: number }>;
}) {
// Global recording uses processId -1
setIsRecording(data.recordings.some(r => r.processId === -1));
}
socket.on('apps:recording', handleRecordingStatus);
return () => {
socket.off('apps:recording', handleRecordingStatus);
};
}, []);
const handleClick = useCallback(() => {
setIsLoading(true);
const endpoint = isRecording ? '/api/global/stop' : '/api/global/record';
fetch(endpoint, { method: 'POST' })
.then(response => {
if (!response.ok) {
throw new Error('Failed to toggle global recording');
}
})
.catch(error => {
console.error('Error toggling global recording:', error);
})
.finally(() => {
setIsLoading(false);
});
}, [isRecording]);
return (
<button
onClick={handleClick}
disabled={isLoading}
className={`
px-4 py-2 rounded-lg font-medium text-sm
transition-colors duration-200
${isLoading ? 'opacity-50 cursor-not-allowed' : ''}
${
isRecording
? 'bg-red-100 text-red-700 hover:bg-red-200'
: 'bg-blue-100 text-blue-700 hover:bg-blue-200'
}
`}
>
<div className="flex items-center gap-2">
{isRecording ? (
<>
<div className="w-2 h-2 rounded-full bg-red-500 animate-pulse" />
Stop Global Recording
</>
) : (
<>
<div className="w-2 h-2 rounded-full bg-blue-500" />
Record System Audio
</>
)}
</div>
</button>
);
}
@@ -1,163 +0,0 @@
import type { ReactElement } from 'react';
export function PlayIcon(): ReactElement {
return (
<svg
className="w-6 h-6 text-gray-900"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M4.5 5.653c0-1.426 1.529-2.33 2.779-1.643l11.54 6.348c1.295.712 1.295 2.573 0 3.285L7.28 19.991c-1.25.687-2.779-.217-2.779-1.643V5.653z"
fill="currentColor"
/>
</svg>
);
}
export function PauseIcon(): ReactElement {
return (
<svg
className="w-6 h-6 text-gray-900"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M6.75 5.25a.75.75 0 01.75-.75H9a.75.75 0 01.75.75v13.5a.75.75 0 01-.75.75H7.5a.75.75 0 01-.75-.75V5.25zm7 0a.75.75 0 01.75-.75h1.5a.75.75 0 01.75.75v13.5a.75.75 0 01-.75.75h-1.5a.75.75 0 01-.75-.75V5.25z"
fill="currentColor"
/>
</svg>
);
}
export function RewindIcon(): ReactElement {
return (
<svg
className="w-5 h-5 text-gray-600"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M12.066 11.2a1 1 0 000 1.6l5.334 4A1 1 0 0019 16V8a1 1 0 00-1.6-.8l-5.334 4zM11 8a1 1 0 00-1.6-.8l-5.334 4a1 1 0 000 1.6l5.334 4A1 1 0 0011 16V8z"
fill="currentColor"
/>
</svg>
);
}
export function ForwardIcon(): ReactElement {
return (
<svg
className="w-5 h-5 text-gray-600"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M5 8a1 1 0 011.6-.8l5.334 4a1 1 0 010 1.6L6.6 16.8A1 1 0 015 16V8zm7.066-.8a1 1 0 00-1.6.8v8a1 1 0 001.6.8l5.334-4a1 1 0 000-1.6l-5.334-4z"
fill="currentColor"
/>
</svg>
);
}
export function DeleteIcon(): ReactElement {
return (
<svg
className="w-5 h-5"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
/>
</svg>
);
}
export function LoadingSpinner(): ReactElement {
return (
<svg className="animate-spin h-4 w-4" viewBox="0 0 24 24">
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
fill="none"
/>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
/>
</svg>
);
}
export function ErrorIcon(): ReactElement {
return (
<svg
className="w-4 h-4 mr-1.5 flex-shrink-0"
fill="currentColor"
viewBox="0 0 20 20"
>
<path
fillRule="evenodd"
d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
clipRule="evenodd"
/>
</svg>
);
}
export function MicrophoneIcon(): ReactElement {
return (
<svg
className="w-4 h-4 mr-1.5 text-blue-500"
viewBox="0 0 20 20"
fill="currentColor"
>
<path d="M7 4a3 3 0 016 0v4a3 3 0 11-6 0V4zm4 10.93A7.001 7.001 0 0017 8a1 1 0 10-2 0A5 5 0 015 8a1 1 0 00-2 0 7.001 7.001 0 006 6.93V17H6a1 1 0 100 2h8a1 1 0 100-2h-3v-2.07z" />
</svg>
);
}
export function WarningIcon(): ReactElement {
return (
<svg className="w-4 h-4 mr-1.5" viewBox="0 0 20 20" fill="currentColor">
<path
fillRule="evenodd"
d="M8.257 3.099c.765-1.36 2.722-1.36 3.486 0l5.58 9.92c.75 1.334-.213 2.98-1.742 2.98H4.42c-1.53 0-2.493-1.646-1.743-2.98l5.58-9.92zM11 13a1 1 0 11-2 0 1 1 0 012 0zm-1-8a1 1 0 00-1 1v3a1 1 0 002 0V6a1 1 0 00-1-1z"
clipRule="evenodd"
/>
</svg>
);
}
export function DefaultAppIcon(): ReactElement {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
className="h-6 w-6"
viewBox="0 0 20 20"
fill="currentColor"
>
<path d="M10 2a3 3 0 00-3 3v4a3 3 0 006 0V5a3 3 0 00-3-3zm0 2a1 1 0 011 1v4a1 1 0 11-2 0V5a1 1 0 011-1z" />
<path d="M3 10a7 7 0 1014 0h-2a5 5 0 11-10 0H3z" />
</svg>
);
}
@@ -1,895 +0,0 @@
import type { ReactElement } from 'react';
import React from 'react';
import ReactMarkdown from 'react-markdown';
import type { SavedRecording, TranscriptionMetadata } from '../types';
import { formatDuration, socket } from '../utils';
import {
DefaultAppIcon,
DeleteIcon,
ErrorIcon,
ForwardIcon,
LoadingSpinner,
MicrophoneIcon,
PauseIcon,
PlayIcon,
RewindIcon,
WarningIcon,
} from './icons';
interface SavedRecordingItemProps {
recording: SavedRecording;
}
// Audio player controls component
function AudioControls({
audioRef,
playbackRate,
onPlaybackRateChange,
onSeek,
onPlayPause,
}: {
audioRef: React.RefObject<HTMLAudioElement | null>;
playbackRate: number;
onPlaybackRateChange: () => void;
onSeek: (seconds: number) => void;
onPlayPause: () => void;
}): ReactElement {
const [currentTime, setCurrentTime] = React.useState('00:00');
const [duration, setDuration] = React.useState('00:00');
React.useEffect(() => {
const audio = audioRef.current;
if (!audio) return;
const formatTime = (time: number) => {
const minutes = Math.floor(time / 60);
const seconds = Math.floor(time % 60);
return `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`;
};
const updateTime = () => {
setCurrentTime(formatTime(audio.currentTime));
setDuration(formatTime(audio.duration));
};
audio.addEventListener('timeupdate', updateTime);
audio.addEventListener('loadedmetadata', updateTime);
return () => {
audio.removeEventListener('timeupdate', updateTime);
audio.removeEventListener('loadedmetadata', updateTime);
};
}, [audioRef]);
return (
<div className="flex items-center justify-between">
<div className="flex items-center space-x-2">
<button
onClick={() => onSeek(-15)}
className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"
title="Back 15 seconds"
>
<RewindIcon />
</button>
<button
onClick={onPlayPause}
className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"
>
{audioRef.current?.paused ? <PlayIcon /> : <PauseIcon />}
</button>
<button
onClick={() => onSeek(30)}
className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"
title="Forward 30 seconds"
>
<ForwardIcon />
</button>
<div className="text-sm font-mono text-gray-500 ml-2">
{currentTime} <span className="text-gray-400">/</span> {duration}
</div>
</div>
<button
onClick={onPlaybackRateChange}
className="px-3 py-1.5 text-sm font-medium text-gray-600 bg-gray-50 hover:bg-gray-100 rounded-lg transition-all duration-200 border border-gray-100 hover:shadow-sm"
>
{playbackRate}x
</button>
</div>
);
}
// Waveform visualization component
function WaveformVisualizer({
containerRef,
waveformData,
currentTime,
fileName,
}: {
containerRef: React.RefObject<HTMLDivElement | null>;
waveformData: number[];
currentTime: number;
fileName: string;
}): ReactElement {
return (
<div
className="relative h-14 bg-gray-50 overflow-hidden rounded-lg border border-gray-100"
ref={containerRef}
>
<div className="absolute inset-0 flex items-end">
{waveformData.map((amplitude, i) => (
<div
key={`${fileName}-bar-${i}`}
className="flex-1 bg-red-400 transition-all duration-200"
style={{
height: `${Math.max(amplitude * 100, 3)}%`,
opacity:
i < Math.floor(currentTime * waveformData.length) ? 1 : 0.3,
margin: '0 0.5px',
}}
/>
))}
</div>
</div>
);
}
// Update TranscriptionMessage component
function TranscriptionMessage({
item,
isNewSpeaker,
isCurrentMessage,
}: {
item: {
speaker: string;
start_time: string;
transcription: string;
};
isNewSpeaker: boolean;
isCurrentMessage: boolean;
}): ReactElement {
return (
<div className="flex items-start gap-3 group transition-all duration-300 w-full">
<div className="w-[120px] flex-shrink-0">
<div className="flex flex-col items-start gap-1">
{isNewSpeaker && (
<div
className={`px-2.5 py-1 rounded-lg text-xs font-medium border transition-colors duration-300 ${
isCurrentMessage
? 'bg-blue-100 text-blue-700 border-blue-200'
: 'bg-blue-50 text-blue-600 border-blue-100'
}`}
>
{item.speaker}
</div>
)}
<div
className={`text-[11px] font-mono ml-2 transition-colors duration-300 ${
isCurrentMessage ? 'text-blue-500' : 'text-gray-400'
}`}
>
{item.start_time}
</div>
</div>
</div>
<div className="flex-1 min-w-0 w-full">
<div
className={`text-sm leading-relaxed rounded-xl px-4 py-2 border transition-all inline-flex duration-300 ${
isCurrentMessage
? 'bg-blue-50/50 text-blue-900 border-blue-200 shadow-md'
: 'bg-white text-gray-600 border-gray-100 shadow-sm hover:shadow-md'
}`}
>
{item.transcription}
</div>
</div>
</div>
);
}
// Add new Summary component
function TranscriptionSummary({ summary }: { summary: string }): ReactElement {
return (
<div className="mb-6 bg-blue-50/50 rounded-xl p-4 border border-blue-100">
<div className="text-xs font-medium text-blue-600 mb-2 uppercase tracking-wider">
Summary
</div>
<div className="text-sm text-gray-700 leading-relaxed prose prose-sm max-w-none prose-headings:text-gray-900 prose-a:text-blue-600 whitespace-pre-wrap">
<ReactMarkdown>{summary}</ReactMarkdown>
</div>
</div>
);
}
// Update TranscriptionContent component
function TranscriptionContent({
transcriptionData,
currentAudioTime,
}: {
transcriptionData: {
segments: Array<{
speaker: string;
start_time: string;
transcription: string;
}>;
summary: string;
title: string;
};
currentAudioTime: number;
}): ReactElement {
const parseTimestamp = (timestamp: string) => {
// Handle "MM:SS" format (without hours)
const [minutes, seconds] = timestamp.split(':');
return parseInt(minutes, 10) * 60 + parseInt(seconds, 10);
};
return (
<div className="space-y-2 py-2 max-h-[400px] overflow-y-auto pr-2 scrollbar-thin scrollbar-thumb-gray-300 scrollbar-track-transparent hover:scrollbar-thumb-gray-400 w-full">
<TranscriptionSummary summary={transcriptionData.summary} />
{transcriptionData.segments.map((item, index) => {
const isNewSpeaker =
index === 0 ||
transcriptionData.segments[index - 1].speaker !== item.speaker;
const startTime = parseTimestamp(item.start_time);
// Use next segment's start time as end time, or add 3 seconds for the last segment
const endTime =
index < transcriptionData.segments.length - 1
? parseTimestamp(transcriptionData.segments[index + 1].start_time)
: startTime + 3;
const isCurrentMessage =
currentAudioTime >= startTime && currentAudioTime < endTime;
return (
<TranscriptionMessage
key={`${item.speaker}-${item.start_time}-${index}`}
item={item}
isNewSpeaker={isNewSpeaker}
isCurrentMessage={isCurrentMessage}
/>
);
})}
</div>
);
}
// Update TranscriptionStatus component
function TranscriptionStatus({
transcription,
transcriptionError,
currentAudioTime,
}: {
transcription?: TranscriptionMetadata;
transcriptionError: string | null;
currentAudioTime: number;
}): ReactElement | null {
if (!transcription && !transcriptionError) {
return null;
}
if (transcription?.transcriptionStatus === 'pending') {
return (
<div className="my-2">
<div className="text-sm text-gray-600 bg-gray-50/50 p-4 border border-gray-100 w-full">
<div className="font-medium text-gray-900 mb-4 flex items-center sticky top-0 bg-gray-50/50 backdrop-blur-sm z-10 py-2">
<MicrophoneIcon />
<span>Processing Audio</span>
</div>
<div className="flex items-center justify-center py-8">
<div className="flex flex-col items-center gap-3">
<LoadingSpinner />
<div className="text-sm text-gray-600">
<span className="font-medium">Starting transcription</span>
<span className="text-gray-400 animate-pulse">...</span>
</div>
<div className="text-xs text-gray-400 max-w-sm text-center">
This may take a few moments depending on the length of the
recording
</div>
</div>
</div>
</div>
</div>
);
}
if (transcriptionError) {
return (
<div className="text-xs text-red-500 m-2 flex items-center bg-red-50 rounded-lg p-2 border border-red-100">
<ErrorIcon />
{transcriptionError}
</div>
);
}
if (
transcription?.transcriptionStatus === 'completed' &&
transcription.transcription
) {
try {
const transcriptionData = transcription.transcription;
if (
!transcriptionData.segments ||
!Array.isArray(transcriptionData.segments)
) {
throw new Error('Invalid transcription data format');
}
return (
<div className="my-2">
<div className="text-sm text-gray-600 bg-gray-50/50 p-4 border border-gray-100 w-full">
<div className="font-medium text-gray-900 mb-4 flex items-center sticky top-0 bg-gray-50/50 backdrop-blur-sm z-10 py-2">
<MicrophoneIcon />
<span>Conversation Transcript</span>
</div>
{transcriptionData.title && (
<div className="mb-4 bg-blue-50/50 rounded-lg p-3 border border-blue-100">
<div className="text-xs font-medium text-blue-600 uppercase tracking-wider mb-1">
Title
</div>
<div className="text-base font-medium text-gray-900">
{transcriptionData.title}
</div>
</div>
)}
<TranscriptionContent
transcriptionData={transcriptionData}
currentAudioTime={currentAudioTime}
/>
</div>
</div>
);
} catch (error) {
return (
<div className="text-sm text-red-500 bg-red-50 rounded-lg p-2 border border-red-100 m-2">
{error instanceof Error
? error.message
: 'Failed to parse transcription data'}
</div>
);
}
}
return null;
}
// Add new RecordingHeader component
function RecordingHeader({
metadata,
fileName,
recordingDate,
duration,
error,
isDeleting,
showDeleteConfirm,
setShowDeleteConfirm,
handleDeleteClick,
}: {
metadata: SavedRecording['metadata'];
fileName: string;
recordingDate: string;
duration: string;
error: string | null;
isDeleting: boolean;
showDeleteConfirm: boolean;
setShowDeleteConfirm: (show: boolean) => void;
handleDeleteClick: () => void;
transcriptionError: string | null;
}): ReactElement {
const [imgError, setImgError] = React.useState(false);
const isGlobalRecording = metadata?.isGlobal;
return (
<div className="flex items-start space-x-4 p-4 bg-gray-50/30">
<div className="relative w-12 h-12 flex-shrink-0">
{!imgError && !isGlobalRecording ? (
<img
src={`/api/recordings/${fileName}/icon.png`}
alt={metadata?.appName || 'Unknown Application'}
className="w-12 h-12 object-contain rounded-lg bg-gray-50 border border-gray-100 shadow-sm transition-transform duration-200 hover:scale-105"
onError={() => setImgError(true)}
/>
) : (
<div className="w-12 h-12 rounded-xl flex items-center justify-center text-gray-500 bg-gray-50 border border-gray-100 shadow-sm">
<DefaultAppIcon />
</div>
)}
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center justify-between">
<div className="flex items-center space-x-2">
<span className="text-gray-900 font-semibold text-base truncate">
{metadata?.appName || 'Unknown Application'}
</span>
{isGlobalRecording && (
<span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">
System Audio
</span>
)}
<span className="text-xs px-2 py-0.5 bg-gray-50 rounded-full text-gray-600 font-medium border border-gray-100">
{duration}
</span>
</div>
<div className="flex items-center">
{showDeleteConfirm ? (
<div className="flex items-center space-x-2">
<button
onClick={() => setShowDeleteConfirm(false)}
className="h-8 px-3 text-sm font-medium text-gray-600 hover:bg-gray-50 rounded-lg transition-colors border border-gray-100"
disabled={isDeleting}
>
Cancel
</button>
<button
onClick={handleDeleteClick}
className="h-8 px-3 text-sm font-medium text-red-600 hover:bg-red-50 rounded-lg transition-colors border border-red-100 disabled:opacity-50 disabled:cursor-not-allowed"
disabled={isDeleting}
>
{isDeleting ? (
<div className="flex items-center space-x-2">
<LoadingSpinner />
<span>Deleting...</span>
</div>
) : (
'Confirm'
)}
</button>
</div>
) : (
<button
onClick={() => setShowDeleteConfirm(true)}
className="h-8 w-8 flex items-center justify-center text-gray-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-colors"
title="Delete recording"
>
<DeleteIcon />
</button>
)}
</div>
</div>
<div className="text-sm text-gray-600 mt-1">{recordingDate}</div>
<div className="text-xs text-gray-400 font-mono mt-0.5 truncate">
{metadata?.bundleIdentifier || fileName}
</div>
{error && (
<div className="text-xs text-red-500 mt-2 flex items-center bg-red-50 rounded-lg p-2 border border-red-100">
<ErrorIcon />
{error}
</div>
)}
</div>
</div>
);
}
// Add new AudioPlayer component
function AudioPlayer({
isLoading,
error,
audioRef,
playbackRate,
handlePlaybackRateChange,
handleSeek,
handlePlayPause,
containerRef,
waveformData,
currentTime,
fileName,
}: {
isLoading: boolean;
error: string | null;
audioRef: React.RefObject<HTMLAudioElement>;
playbackRate: number;
handlePlaybackRateChange: () => void;
handleSeek: (seconds: number) => void;
handlePlayPause: () => void;
containerRef: React.RefObject<HTMLDivElement>;
waveformData: number[];
currentTime: number;
fileName: string;
}): ReactElement {
return (
<div className="px-4 pb-4">
{isLoading && !error ? (
<div className="h-14 bg-gray-50 rounded-lg flex items-center justify-center border border-gray-100">
<LoadingSpinner />
<span className="ml-2 text-sm text-gray-600 font-medium">
Loading audio...
</span>
</div>
) : (
<div className="flex flex-col space-y-3">
<AudioControls
audioRef={audioRef}
playbackRate={playbackRate}
onPlaybackRateChange={handlePlaybackRateChange}
onSeek={handleSeek}
onPlayPause={handlePlayPause}
/>
<WaveformVisualizer
containerRef={containerRef}
waveformData={waveformData}
currentTime={currentTime}
fileName={fileName}
/>
</div>
)}
</div>
);
}
// Add new TranscribeButton component
function TranscribeButton({
transcriptionStatus,
onTranscribe,
}: {
transcriptionStatus?: TranscriptionMetadata['transcriptionStatus'];
onTranscribe: () => void;
}): ReactElement {
return (
<div className="px-4 pb-4">
<div className="flex justify-end">
<button
onClick={onTranscribe}
disabled={transcriptionStatus === 'pending'}
className={`h-8 px-3 text-sm font-medium rounded-lg transition-colors border flex items-center space-x-2
${
transcriptionStatus === 'pending'
? 'bg-blue-50 text-blue-600 border-blue-200 cursor-not-allowed'
: transcriptionStatus === 'completed'
? 'text-blue-600 hover:bg-blue-50 border-blue-100'
: transcriptionStatus === 'error'
? 'text-red-600 hover:bg-red-50 border-red-100'
: 'text-blue-600 hover:bg-blue-50 border-blue-100'
}`}
>
{transcriptionStatus === 'pending' ? (
<>
<LoadingSpinner />
<span>Transcribing...</span>
</>
) : transcriptionStatus === 'completed' ? (
<>
<MicrophoneIcon />
<span>Transcribe Again</span>
</>
) : transcriptionStatus === 'error' ? (
<>
<WarningIcon />
<span>Retry Transcription</span>
</>
) : (
<>
<MicrophoneIcon />
<span>Transcribe</span>
</>
)}
</button>
</div>
</div>
);
}
// Main SavedRecordingItem component (simplified)
export function SavedRecordingItem({
recording,
}: SavedRecordingItemProps): ReactElement {
const [error, setError] = React.useState<string | null>(null);
const [isLoading, setIsLoading] = React.useState(true);
const [isDeleting, setIsDeleting] = React.useState(false);
const [showDeleteConfirm, setShowDeleteConfirm] = React.useState(false);
const [playbackRate, setPlaybackRate] = React.useState(1);
const [waveformData, setWaveformData] = React.useState<number[]>([]);
const [currentTime, setCurrentTime] = React.useState(0);
const audioRef = React.useRef<HTMLAudioElement | null>(null);
const containerRef = React.useRef<HTMLDivElement | null>(null);
const [segments, setSegments] = React.useState(40);
const [currentAudioTime, setCurrentAudioTime] = React.useState(0);
const [transcriptionError, setTranscriptionError] = React.useState<
string | null
>(null);
const metadata = recording.metadata;
// Ensure we have a valid filename, fallback to an empty string if undefined
const fileName = recording.wav || '';
const recordingDate = metadata
? new Date(metadata.recordingStartTime).toLocaleString()
: 'Unknown date';
const duration = metadata
? formatDuration(metadata.recordingDuration * 1000)
: 'Unknown duration';
// Update current audio time
React.useEffect(() => {
const audio = audioRef.current;
if (audio) {
const handleTimeUpdate = () => {
setCurrentAudioTime(audio.currentTime);
};
audio.addEventListener('timeupdate', handleTimeUpdate);
return () => audio.removeEventListener('timeupdate', handleTimeUpdate);
}
return () => {};
}, []);
// Calculate number of segments based on container width
React.useEffect(() => {
const updateSegments = () => {
if (containerRef.current) {
// Each bar should be at least 2px wide (1px bar + 1px gap)
const width = containerRef.current.offsetWidth;
setSegments(Math.floor(width / 2));
}
};
updateSegments();
const resizeObserver = new ResizeObserver(updateSegments);
if (containerRef.current) {
resizeObserver.observe(containerRef.current);
}
return () => resizeObserver.disconnect();
}, []);
const processAudioData = React.useCallback(async () => {
try {
// Check if fileName is empty
if (!fileName) {
throw new Error('Invalid recording filename');
}
const response = await fetch(`/api/recordings/${fileName}/recording.wav`);
if (!response.ok) {
throw new Error(
`Failed to fetch audio file (${response.status}): ${response.statusText}`
);
}
const audioContext = new AudioContext();
const arrayBuffer = await response.arrayBuffer();
// Ensure we have data to process
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
throw new Error('No audio data received');
}
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
const channelData = audioBuffer.getChannelData(0);
// Process the audio data in chunks to create the waveform
const numberOfSamples = channelData.length;
const samplesPerSegment = Math.floor(numberOfSamples / segments);
const waveform: number[] = [];
for (let i = 0; i < segments; i++) {
const start = i * samplesPerSegment;
const end = start + samplesPerSegment;
const segmentData = channelData.slice(start, end);
// Calculate RMS (root mean square) for better amplitude representation
const rms = Math.sqrt(
segmentData.reduce((sum, sample) => sum + sample * sample, 0) /
segmentData.length
);
waveform.push(rms);
}
// Normalize the waveform data to a 0-1 range
const maxAmplitude = Math.max(...waveform);
const normalizedWaveform = waveform.map(amp => amp / maxAmplitude);
setWaveformData(normalizedWaveform);
setIsLoading(false);
} catch (err) {
console.error('Error processing audio:', err);
setError(
err instanceof Error ? err.message : 'Failed to process audio data'
);
setIsLoading(false);
}
}, [fileName, segments]);
React.useEffect(() => {
const audio = audioRef.current;
if (audio) {
const handleError = (e: ErrorEvent) => {
console.error('Audio error:', e);
setError('Failed to load audio');
setIsLoading(false);
};
const handleLoadedMetadata = () => {
void processAudioData().catch(err => {
console.error('Error processing audio data:', err);
setError('Failed to process audio data');
setIsLoading(false);
});
};
const handleTimeUpdate = () => {
setCurrentTime(audio.currentTime / audio.duration);
};
audio.addEventListener('error', handleError as EventListener);
audio.addEventListener('loadedmetadata', handleLoadedMetadata);
audio.addEventListener('timeupdate', handleTimeUpdate);
return () => {
audio.removeEventListener('error', handleError as EventListener);
audio.removeEventListener('loadedmetadata', handleLoadedMetadata);
audio.removeEventListener('timeupdate', handleTimeUpdate);
};
}
return () => {};
}, [processAudioData]);
const handlePlayPause = React.useCallback(() => {
if (audioRef.current) {
if (audioRef.current.paused) {
// oxlint-disable-next-line @typescript-eslint/no-floating-promises
void audioRef.current.play();
} else {
audioRef.current.pause();
}
}
}, []);
const handleSeek = React.useCallback((seconds: number) => {
if (audioRef.current) {
audioRef.current.currentTime += seconds;
}
}, []);
const handlePlaybackRateChange = React.useCallback(() => {
if (audioRef.current) {
const newRate = playbackRate === 1 ? 1.5 : 1;
audioRef.current.playbackRate = newRate;
setPlaybackRate(newRate);
}
}, [playbackRate]);
const handleDelete = React.useCallback(async () => {
setIsDeleting(true);
setError(null); // Clear any previous errors
try {
// Check if filename is valid
if (!recording.wav) {
throw new Error('Invalid recording filename');
}
const response = await fetch(`/api/recordings/${recording.wav}`, {
method: 'DELETE',
});
if (!response.ok) {
let errorMessage: string;
try {
const errorData = await response.json();
errorMessage = errorData.error;
} catch {
errorMessage = `Server error (${response.status}): ${response.statusText}`;
}
throw new Error(errorMessage);
}
setShowDeleteConfirm(false);
} catch (err) {
console.error('Error deleting recording:', err);
setError(
err instanceof Error ? err.message : 'An unexpected error occurred'
);
} finally {
setIsDeleting(false);
}
}, [recording.wav]);
const handleDeleteClick = React.useCallback(() => {
void handleDelete().catch(err => {
console.error('Unexpected error during deletion:', err);
setError('An unexpected error occurred');
});
}, [handleDelete]);
React.useEffect(() => {
// Listen for transcription events
socket.on(
'apps:recording-transcription-start',
(data: { filename: string }) => {
if (recording.wav && data.filename === recording.wav) {
setTranscriptionError(null);
}
}
);
socket.on(
'apps:recording-transcription-end',
(data: {
filename: string;
success: boolean;
transcription?: string;
error?: string;
}) => {
if (recording.wav && data.filename === recording.wav && !data.success) {
setTranscriptionError(data.error || 'Transcription failed');
}
}
);
return () => {
socket.off('apps:recording-transcription-start');
socket.off('apps:recording-transcription-end');
};
}, [recording.wav]);
const handleTranscribe = React.useCallback(async () => {
try {
// Check if filename is valid
if (!recording.wav) {
throw new Error('Invalid recording filename');
}
const response = await fetch(
`/api/recordings/${recording.wav}/transcribe`,
{
method: 'POST',
}
);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to start transcription');
}
} catch (err) {
setTranscriptionError(
err instanceof Error ? err.message : 'Failed to start transcription'
);
}
}, [recording.wav]);
return (
<div className="bg-white rounded-lg shadow-sm hover:shadow-md transition-all duration-300 overflow-hidden mb-3 border border-gray-100 hover:border-gray-200">
<RecordingHeader
metadata={metadata}
fileName={fileName}
recordingDate={recordingDate}
duration={duration}
error={error}
isDeleting={isDeleting}
showDeleteConfirm={showDeleteConfirm}
setShowDeleteConfirm={setShowDeleteConfirm}
handleDeleteClick={handleDeleteClick}
transcriptionError={transcriptionError}
/>
<AudioPlayer
isLoading={isLoading}
error={error}
audioRef={audioRef as React.RefObject<HTMLAudioElement>}
playbackRate={playbackRate}
handlePlaybackRateChange={handlePlaybackRateChange}
handleSeek={handleSeek}
handlePlayPause={handlePlayPause}
containerRef={containerRef as React.RefObject<HTMLDivElement>}
waveformData={waveformData}
currentTime={currentTime}
fileName={fileName}
/>
<audio
ref={audioRef}
src={fileName ? `/api/recordings/${fileName}/recording.wav` : ''}
preload="metadata"
className="hidden"
/>
<TranscriptionStatus
transcription={recording.transcription}
transcriptionError={transcriptionError}
currentAudioTime={currentAudioTime}
/>
<TranscribeButton
transcriptionStatus={recording.transcription?.transcriptionStatus}
onTranscribe={() => void handleTranscribe()}
/>
</div>
);
}
@@ -1,41 +0,0 @@
import useSWRSubscription from 'swr/subscription';
import type { SavedRecording } from '../types';
import { socket } from '../utils';
import { SavedRecordingItem } from './saved-recording-item';
export function SavedRecordings(): React.ReactElement {
const { data: recordings = [] } = useSWRSubscription<SavedRecording[]>(
'saved-recordings',
(
_key: string,
{ next }: { next: (err: Error | null, data?: SavedRecording[]) => void }
) => {
// Subscribe to saved recordings updates
socket.on('apps:saved', (data: { recordings: SavedRecording[] }) => {
next(null, data.recordings);
});
fetch('/api/apps/saved')
.then(res => res.json())
.then(data => next(null, data.recordings))
.catch(err => next(err));
return () => {
socket.off('apps:saved');
};
}
);
if (recordings.length === 0) {
return <p className="text-gray-500 italic text-sm">No saved recordings</p>;
}
return (
<div className="space-y-1">
{recordings.map(recording => (
<SavedRecordingItem key={recording.wav} recording={recording} />
))}
</div>
);
}
@@ -1,11 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Media Capture Playground</title>
</head>
<body>
<div id="root"></div>
</body>
</html>
@@ -1 +0,0 @@
@import 'tailwindcss';
@@ -1,11 +0,0 @@
import './main.css';
import { createRoot } from 'react-dom/client';
import { App } from './app';
const rootElement = document.getElementById('root');
if (!rootElement) {
throw new Error('Failed to find the root element');
}
createRoot(rootElement).render(<App />);
@@ -1,60 +0,0 @@
export interface App {
processId: number;
processGroupId: number;
bundleIdentifier: string;
name: string;
isRunning: boolean;
}
export interface AppGroup {
processGroupId: number;
rootApp: App;
apps: App[];
}
export interface RecordingStatus {
processId: number;
bundleIdentifier: string;
name: string;
startTime: number;
duration: number;
isGlobal?: boolean;
}
export interface RecordingMetadata {
appName: string;
bundleIdentifier: string;
processId: number;
recordingStartTime: number;
recordingEndTime: number;
recordingDuration: number;
sampleRate: number;
channels: number;
totalSamples: number;
icon?: Uint8Array;
mp3: string;
isGlobal?: boolean;
}
export interface TranscriptionMetadata {
transcriptionStartTime: number;
transcriptionEndTime: number;
transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error';
transcription?: {
title: string;
segments: Array<{
speaker: string;
start_time: string;
end_time: string;
transcription: string;
}>;
summary: string;
};
error?: string;
}
export interface SavedRecording {
wav: string;
metadata?: RecordingMetadata;
transcription?: TranscriptionMetadata;
}
@@ -1,19 +0,0 @@
import { io } from 'socket.io-client';
// Create a singleton socket instance
export const socket = io('http://localhost:6544');
export function formatDuration(ms: number): string {
const seconds = Math.floor(ms / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
return `${hours.toString().padStart(2, '0')}:${(minutes % 60)
.toString()
.padStart(2, '0')}:${(seconds % 60).toString().padStart(2, '0')}`;
}
// Helper function to convert timestamp (MM:SS.mmm) to seconds
export function timestampToSeconds(timestamp: string): number {
const [minutes, seconds] = timestamp.split(':').map(parseFloat);
return minutes * 60 + seconds;
}
+5 -3
View File
@@ -9,12 +9,14 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
affine_common = { workspace = true, features = ["hashcash"] }
affine_importer = { workspace = true }
affine_media_capture = { path = "./media_capture" }
affine_nbstore = { workspace = true, features = ["napi"] }
affine_sqlite_v1 = { path = "./sqlite_v1" }
napi = { workspace = true }
napi-derive = { workspace = true }
once_cell = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true, default-features = false, features = [
"chrono",
"macros",
@@ -38,9 +40,9 @@ mimalloc = { workspace = true }
mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
[dev-dependencies]
chrono = { workspace = true }
serde_json = { workspace = true }
uuid = { workspace = true }
chrono = { workspace = true }
uuid = { workspace = true }
zip = { workspace = true }
[build-dependencies]
napi-build = { workspace = true }
@@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url';
import test from 'ava';
import { SqliteConnection, ValidationResult } from '../index';
import { SqliteConnection, ValidationResult } from '../index.js';
test('db validate', async t => {
const path = fileURLToPath(
@@ -1,6 +1,6 @@
import test from 'ava';
import { DocStoragePool } from '../index';
import { DocStoragePool } from '../index.js';
test('can batch read/write pool', async t => {
const pool = new DocStoragePool();
+25 -9
View File
@@ -18,13 +18,6 @@ export declare class ApplicationStateChangedSubscriber {
unsubscribe(): void
}
export declare class AudioCaptureSession {
stop(): void
get sampleRate(): number
get channels(): number
get actualSampleRate(): number
}
export declare class ShareableContent {
static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber
static onAppStateChanged(app: ApplicationInfo, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber
@@ -32,8 +25,6 @@ export declare class ShareableContent {
static applications(): Array<ApplicationInfo>
static applicationWithProcessId(processId: number): ApplicationInfo | null
static isUsingMicrophone(processId: number): boolean
static tapAudio(processId: number, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
static tapGlobalAudio(excludedProcesses: Array<ApplicationInfo> | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
}
export declare function abortRecording(id: string): Promise<void>
@@ -75,6 +66,29 @@ export interface RecordingStartOptions {
export declare function startRecording(opts: RecordingStartOptions): Promise<RecordingSessionMeta>
export declare function stopRecording(id: string): Promise<RecordingArtifact>
export declare function cancelImportSession(sessionId: string): void
export interface CreateImportBatchLimits {
maxDocs?: number
maxBlobs?: number
maxBlobBytes?: number
}
export declare function createImportSession(options: CreateImportSessionOptions): string
export interface CreateImportSessionOptions {
format: string
source: CreateImportSessionSource
batchLimits?: CreateImportBatchLimits
}
export interface CreateImportSessionSource {
kind: string
path: string
}
export declare function disposeImportSession(sessionId: string): void
export interface MermaidRenderOptions {
theme?: string
fontFamily?: string
@@ -92,6 +106,8 @@ export interface MermaidRenderResult {
export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
export declare function nextImportBatch(sessionId: string): string | null
export declare function renderMermaidSvg(request: MermaidRenderRequest): MermaidRenderResult
export declare function renderTypstSvg(request: TypstRenderRequest): TypstRenderResult
+4 -1
View File
@@ -575,14 +575,17 @@ module.exports = nativeBinding
module.exports.ApplicationInfo = nativeBinding.ApplicationInfo
module.exports.ApplicationListChangedSubscriber = nativeBinding.ApplicationListChangedSubscriber
module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStateChangedSubscriber
module.exports.AudioCaptureSession = nativeBinding.AudioCaptureSession
module.exports.ShareableContent = nativeBinding.ShareableContent
module.exports.abortRecording = nativeBinding.abortRecording
module.exports.decodeAudio = nativeBinding.decodeAudio
module.exports.decodeAudioSync = nativeBinding.decodeAudioSync
module.exports.startRecording = nativeBinding.startRecording
module.exports.stopRecording = nativeBinding.stopRecording
module.exports.cancelImportSession = nativeBinding.cancelImportSession
module.exports.createImportSession = nativeBinding.createImportSession
module.exports.disposeImportSession = nativeBinding.disposeImportSession
module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse
module.exports.nextImportBatch = nativeBinding.nextImportBatch
module.exports.renderMermaidSvg = nativeBinding.renderMermaidSvg
module.exports.renderTypstSvg = nativeBinding.renderTypstSvg
module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse
@@ -1,149 +0,0 @@
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
Whisper,
WhisperFullParams,
WhisperSamplingStrategy,
} from '@napi-rs/whisper';
import { BehaviorSubject, EMPTY, Observable } from 'rxjs';
import {
distinctUntilChanged,
exhaustMap,
groupBy,
mergeMap,
switchMap,
tap,
} from 'rxjs/operators';
import { type Application, ShareableContent } from './index.js';
const rootDir = join(fileURLToPath(import.meta.url), '..');
const shareableContent = new ShareableContent();
const appList = new Set([
'com.tinyspeck.slackmacgap.helper',
'us.zoom.xos',
'org.mozilla.firefoxdeveloperedition',
]);
console.info(shareableContent.applications().map(app => app.bundleIdentifier));
const GGLM_LARGE = join(rootDir, 'ggml-large-v3-turbo.bin');
const whisper = new Whisper(GGLM_LARGE, {
useGpu: true,
gpuDevice: 1,
});
const whisperParams = new WhisperFullParams(WhisperSamplingStrategy.Greedy);
const SAMPLE_WINDOW_MS = 3000; // 3 seconds, similar to stream.cpp's step_ms
const SAMPLES_PER_WINDOW = (SAMPLE_WINDOW_MS / 1000) * 16000; // 16kHz sample rate
// eslint-disable-next-line rxjs/finnish
const runningApplications = new BehaviorSubject(
shareableContent.applications()
);
const applicationListChangedSubscriber =
ShareableContent.onApplicationListChanged(() => {
runningApplications.next(shareableContent.applications());
});
runningApplications
.pipe(
mergeMap(apps => apps.filter(app => appList.has(app.bundleIdentifier))),
groupBy(app => app.bundleIdentifier),
mergeMap(app$ =>
app$.pipe(
exhaustMap(app =>
new Observable<[Application, boolean]>(subscriber => {
const stateSubscriber = ShareableContent.onAppStateChanged(
app,
err => {
if (err) {
subscriber.error(err);
return;
}
subscriber.next([app, app.isRunning]);
}
);
return () => {
stateSubscriber.unsubscribe();
};
}).pipe(
distinctUntilChanged(
([_, isRunningA], [__, isRunningB]) => isRunningA === isRunningB
),
switchMap(([app]) =>
!app.isRunning
? EMPTY
: new Observable(observer => {
const buffers: Float32Array[] = [];
const audioStream = app.tapAudio((err, samples) => {
if (err) {
observer.error(err);
return;
}
if (samples) {
buffers.push(samples);
observer.next(samples);
// Calculate total samples in buffer
const totalSamples = buffers.reduce(
(acc, buf) => acc + buf.length,
0
);
// Process when we have enough samples for our window
if (totalSamples >= SAMPLES_PER_WINDOW) {
// Concatenate all buffers
const concatenated = new Float32Array(totalSamples);
let offset = 0;
buffers.forEach(buf => {
concatenated.set(buf, offset);
offset += buf.length;
});
// Transcribe the audio
const result = whisper.full(
whisperParams,
concatenated
);
// Print results
console.info(result);
// Keep any remaining samples for next window
const remainingSamples =
totalSamples - SAMPLES_PER_WINDOW;
if (remainingSamples > 0) {
const lastBuffer = buffers[buffers.length - 1];
buffers.length = 0;
buffers.push(lastBuffer.slice(-remainingSamples));
} else {
buffers.length = 0;
}
}
}
});
return () => {
audioStream.stop();
};
})
)
)
)
)
),
tap({
finalize: () => {
applicationListChangedSubscriber.unsubscribe();
},
})
)
.subscribe();
@@ -682,14 +682,6 @@ impl ShareableContent {
}
}
#[napi]
pub fn tap_audio(
process_id: u32,
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
ShareableContent::tap_audio_with_callback(process_id, AudioCallback::Js(Arc::new(audio_stream_callback)))
}
pub(crate) fn tap_global_audio_with_callback(
excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: AudioCallback,
@@ -706,15 +698,4 @@ impl ShareableContent {
let boxed_manager = Box::new(device_manager);
Ok(AudioCaptureSession::new(boxed_manager))
}
#[napi]
pub fn tap_global_audio(
excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
ShareableContent::tap_global_audio_with_callback(
excluded_processes,
AudioCallback::Js(Arc::new(audio_stream_callback)),
)
}
}
@@ -21,7 +21,6 @@ use coreaudio::sys::{
kAudioSubTapUIDKey,
};
use napi::bindgen_prelude::Result;
use napi_derive::napi;
use objc2::runtime::AnyObject;
use crate::{
@@ -887,8 +886,6 @@ impl Drop for AggregateDeviceManager {
}
}
// NEW NAPI Struct: AudioCaptureSession
#[napi]
pub struct AudioCaptureSession {
// Use Option<Box<...>> to allow taking ownership in stop()
manager: Option<Box<AggregateDeviceManager>>,
@@ -896,9 +893,7 @@ pub struct AudioCaptureSession {
channels: Option<u32>,
}
#[napi]
impl AudioCaptureSession {
// Constructor called internally, not directly via NAPI
pub(crate) fn new(manager: Box<AggregateDeviceManager>) -> Self {
Self {
manager: Some(manager),
@@ -907,7 +902,6 @@ impl AudioCaptureSession {
}
}
#[napi]
pub fn stop(&mut self) -> Result<()> {
if let Some(manager) = self.manager.take() {
// Cache the stats before dropping
@@ -926,7 +920,6 @@ impl AudioCaptureSession {
}
}
#[napi(getter)]
pub fn get_sample_rate(&self) -> Result<f64> {
if let Some(manager) = &self.manager {
manager
@@ -943,7 +936,6 @@ impl AudioCaptureSession {
}
}
#[napi(getter)]
pub fn get_channels(&self) -> Result<u32> {
if let Some(manager) = &self.manager {
manager
@@ -960,7 +952,6 @@ impl AudioCaptureSession {
}
}
#[napi(getter)]
pub fn get_actual_sample_rate(&self) -> Result<f64> {
if let Some(manager) = &self.manager {
manager
@@ -200,7 +200,6 @@ fn emit_mixed_frames(
}
}
#[napi]
pub struct AudioCaptureSession {
mic_stream: Option<cpal::Stream>,
lb_stream: Option<cpal::Stream>,
@@ -258,25 +257,20 @@ where
}
}
#[napi]
impl AudioCaptureSession {
#[napi(getter)]
pub fn get_sample_rate(&self) -> f64 {
self.sample_rate.0 as f64
}
#[napi(getter)]
pub fn get_channels(&self) -> u32 {
self.channels
}
#[napi(getter)]
pub fn get_actual_sample_rate(&self) -> f64 {
// For CPAL we always operate at the target rate which is sample_rate
self.sample_rate.0 as f64
}
#[napi]
pub fn stop(&mut self) -> Result<()> {
teardown_audio_capture_resources(
&mut self.mic_stream,
@@ -225,16 +225,6 @@ impl ShareableContent {
crate::windows::audio_capture::start_recording(audio_stream_callback, target)
}
#[napi]
pub fn tap_audio(
_process_id: u32, // Currently unused - Windows captures global audio
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
// On Windows with CPAL, we capture global audio (mic + loopback)
// since per-application audio tapping isn't supported the same way as macOS
ShareableContent::tap_audio_with_callback(_process_id, AudioCallback::Js(Arc::new(audio_stream_callback)), None)
}
pub(crate) fn tap_global_audio_with_callback(
_excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: AudioCallback,
@@ -246,18 +236,6 @@ impl ShareableContent {
crate::windows::audio_capture::start_recording(audio_stream_callback, target)
}
#[napi]
pub fn tap_global_audio(
_excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
ShareableContent::tap_global_audio_with_callback(
_excluded_processes,
AudioCallback::Js(Arc::new(audio_stream_callback)),
None,
)
}
#[napi]
pub fn is_using_microphone(process_id: u32) -> Result<bool> {
is_process_actively_using_microphone(process_id)
+2 -1
View File
@@ -13,7 +13,8 @@ napi = ["affine_common/napi"]
use-as-lib = ["napi-derive/noop", "napi/noop"]
[dependencies]
affine_common = { workspace = true, features = ["ydoc-loader"] }
affine_common = { workspace = true, features = ["napi"] }
affine_doc_loader = { workspace = true }
affine_schema = { path = "../schema" }
anyhow = { workspace = true }
bincode = { version = "2.0.1", features = ["serde"] }
@@ -1,4 +1,4 @@
use affine_common::doc_parser::ParseError;
use affine_doc_loader::ParseError;
pub type Result<T> = std::result::Result<T, Error>;
@@ -1,4 +1,4 @@
use affine_common::doc_parser::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary};
use affine_doc_loader::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary};
use memory_indexer::{SearchHit, SnapshotData};
use napi_derive::napi;
use serde::Serialize;
@@ -276,7 +276,7 @@ fn merge_updates(mut segments: Vec<Vec<u8>>, guid: &str) -> Result<Vec<u8>> {
mod tests {
use std::path::{Path, PathBuf};
use affine_common::doc_parser::ParseError;
use affine_doc_loader::ParseError;
use chrono::Utc;
use serde_json::Value;
use tokio::fs;
@@ -311,7 +311,7 @@ mod tests {
storage
.set_blob(crate::SetBlob {
key: "large-blob".to_string(),
data: vec![7; 1024 * 1024],
data: Into::<crate::Data>::into(vec![7; 1024 * 1024]),
mime: "application/octet-stream".to_string(),
})
.await
@@ -0,0 +1,270 @@
use std::{
collections::HashMap,
path::PathBuf,
sync::{
Mutex,
atomic::{AtomicU64, Ordering},
},
};
use affine_importer::{
ImportBatchLimits, ImportFormat, ImportOptions, ImportSession, ImportSessionOptions, ImportSessionSource,
};
use napi::{Status, bindgen_prelude::*};
use napi_derive::napi;
use once_cell::sync::Lazy;
static IMPORT_SESSIONS: Lazy<Mutex<HashMap<String, ImportSession>>> = Lazy::new(|| Mutex::new(HashMap::new()));
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
#[napi(object)]
pub struct CreateImportSessionOptions {
pub format: String,
pub source: CreateImportSessionSource,
pub batch_limits: Option<CreateImportBatchLimits>,
}
#[napi(object)]
pub struct CreateImportSessionSource {
pub kind: String,
pub path: String,
}
#[napi(object)]
pub struct CreateImportBatchLimits {
pub max_docs: Option<u32>,
pub max_blobs: Option<u32>,
pub max_blob_bytes: Option<i64>,
}
fn parse_format(format: &str) -> Result<ImportFormat> {
match format {
"markdownZip" => Ok(ImportFormat::MarkdownZip),
"notionZip" => Ok(ImportFormat::NotionZip),
"notionMarkdownZip" => Ok(ImportFormat::NotionMarkdownZip),
"notionHtmlZip" => Ok(ImportFormat::NotionHtmlZip),
"obsidian" => Ok(ImportFormat::Obsidian),
"bearZip" => Ok(ImportFormat::BearZip),
_ => Err(Error::new(
Status::InvalidArg,
format!("unsupported import format: {format}"),
)),
}
}
fn map_import_error(error: affine_importer::ImportError) -> Error {
let status = match error {
affine_importer::ImportError::Cancelled => Status::Cancelled,
affine_importer::ImportError::UnsupportedFormat | affine_importer::ImportError::InvalidSource(_) => {
Status::InvalidArg
}
affine_importer::ImportError::Zip(_)
| affine_importer::ImportError::Io(_)
| affine_importer::ImportError::Document(_) => Status::GenericFailure,
};
Error::new(status, error.to_string())
}
#[napi]
pub fn create_import_session(options: CreateImportSessionOptions) -> Result<String> {
let format = parse_format(&options.format)?;
let source = match options.source.kind.as_str() {
"filePath" => ImportSessionSource::FilePath(PathBuf::from(options.source.path)),
"directoryPath" => ImportSessionSource::DirectoryPath(PathBuf::from(options.source.path)),
kind => {
return Err(Error::new(
Status::InvalidArg,
format!("unsupported import source kind: {kind}"),
));
}
};
let mut batch_limits = ImportBatchLimits::default();
if let Some(limits) = options.batch_limits {
if let Some(max_docs) = limits.max_docs {
batch_limits.max_docs = max_docs as usize;
}
if let Some(max_blobs) = limits.max_blobs {
batch_limits.max_blobs = max_blobs as usize;
}
if let Some(max_blob_bytes) = limits.max_blob_bytes {
if !(0..=100 * 1024 * 1024).contains(&max_blob_bytes) {
return Err(Error::new(Status::InvalidArg, "maxBlobBytes not valid"));
}
batch_limits.max_blob_bytes = max_blob_bytes as u64;
}
}
let session = ImportSession::create(ImportSessionOptions {
format,
source,
import_options: ImportOptions::default(),
batch_limits,
})
.map_err(map_import_error)?;
let id = format!("import-session-{}", NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed));
IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?
.insert(id.clone(), session);
Ok(id)
}
#[napi]
pub fn next_import_batch(session_id: String) -> Result<Option<String>> {
let mut sessions = IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?;
let session = sessions
.get_mut(&session_id)
.ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?;
session
.next_batch()
.map_err(map_import_error)?
.map(|batch| serde_json::to_string(&batch).map_err(|err| Error::new(Status::GenericFailure, err.to_string())))
.transpose()
}
#[napi]
pub fn cancel_import_session(session_id: String) -> Result<()> {
let mut sessions = IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?;
let session = sessions
.get_mut(&session_id)
.ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?;
session.cancel();
Ok(())
}
#[napi]
pub fn dispose_import_session(session_id: String) -> Result<()> {
IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?
.remove(&session_id);
Ok(())
}
#[cfg(test)]
mod tests {
use std::{
fs,
io::Write,
path::PathBuf,
sync::atomic::{AtomicU64, Ordering},
};
use zip::{ZipWriter, write::SimpleFileOptions};
use super::*;
static NEXT_TEST_ARCHIVE_ID: AtomicU64 = AtomicU64::new(1);
fn archive_path(entries: &[(&str, &[u8])]) -> PathBuf {
let path = std::env::temp_dir().join(format!(
"affine-native-import-{}-{}.zip",
std::process::id(),
NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed)
));
let file = std::fs::File::create(&path).unwrap();
let mut writer = ZipWriter::new(file);
for (path, bytes) in entries {
writer.start_file(path, SimpleFileOptions::default()).unwrap();
writer.write_all(bytes).unwrap();
}
writer.finish().unwrap();
path
}
fn directory_path(entries: &[(&str, &[u8])]) -> PathBuf {
let root = std::env::temp_dir().join(format!(
"affine-native-import-dir-{}-{}",
std::process::id(),
NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed)
));
for (path, bytes) in entries {
let path = root.join(path);
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, bytes).unwrap();
}
root
}
#[test]
fn import_session_returns_one_serialized_batch() {
let path = archive_path(&[("entry.md", b"entry")]);
let id = create_import_session(CreateImportSessionOptions {
format: "markdownZip".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
})
.unwrap();
let batch = next_import_batch(id.clone()).unwrap().unwrap();
assert!(batch.contains("\"docs\""));
assert!(next_import_batch(id.clone()).unwrap().is_none());
dispose_import_session(id).unwrap();
let _ = std::fs::remove_file(path);
}
#[test]
fn import_session_cancel_blocks_next_batch() {
let path = archive_path(&[("entry.md", b"entry")]);
let id = create_import_session(CreateImportSessionOptions {
format: "markdownZip".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
})
.unwrap();
cancel_import_session(id.clone()).unwrap();
assert!(next_import_batch(id.clone()).is_err());
dispose_import_session(id).unwrap();
let _ = std::fs::remove_file(path);
}
#[test]
fn import_session_accepts_directory_source() {
let path = directory_path(&[("entry.md", b"entry")]);
let id = create_import_session(CreateImportSessionOptions {
format: "obsidian".to_string(),
source: CreateImportSessionSource {
kind: "directoryPath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
})
.unwrap();
let batch = next_import_batch(id.clone()).unwrap().unwrap();
assert!(batch.contains("\"docs\""));
assert!(next_import_batch(id.clone()).unwrap().is_none());
dispose_import_session(id).unwrap();
let _ = fs::remove_dir_all(path);
}
#[test]
fn import_session_rejects_negative_blob_byte_limit() {
let path = archive_path(&[("entry.md", b"entry")]);
let result = create_import_session(CreateImportSessionOptions {
format: "markdownZip".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: Some(CreateImportBatchLimits {
max_docs: None,
max_blobs: None,
max_blob_bytes: Some(-1),
}),
});
assert!(result.is_err());
let _ = std::fs::remove_file(path);
}
}
+2
View File
@@ -1,4 +1,5 @@
pub mod hashcash;
mod import_session;
#[cfg(not(any(target_os = "android", target_os = "ios")))]
pub mod preview;
@@ -10,3 +11,4 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
pub use affine_media_capture::*;
pub use affine_nbstore::*;
pub use affine_sqlite_v1::*;
pub use import_session::*;