Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-11 15:16:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): import progress & perf (#15197)

Browse Source 
#### PR Dependency Tree


* **PR #15197** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a new import pipeline with “plan then commit” batch handling for
Markdown, Notion HTML, Obsidian, and Bear backups.
* Enabled native import sessions with progress, cancellation, and
batch-by-batch committing (including assets, folders, icons, and tags).
  * Added web preflight limits for ZIP and multi-file imports.
* **Bug Fixes**
* Improved import error/warning reporting and continued processing when
some items fail.
* Strengthened snapshot-based file/directory picking to preserve paths.
* **Chores**
  * Updated project packaging/configuration for the new import workflow.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-07-06 01:29:24 +08:00
committed by GitHub
parent 477015f064
commit 8d72e4dc29
106 changed files with 3329 additions and 11690 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.taplo.toml
+1
View File
@@ -2,6 +2,7 @@ exclude = [
"node_modules/**/*.toml",
"target/**/*.toml",
"packages/frontend/apps/ios/App/Packages/AffineGraphQL/**/*.toml",
"packages/frontend/apps/ios/App/Packages/Intelligents/**/*.toml",
]
# https://taplo.tamasfe.dev/configuration/formatter-options.html
Cargo.lock
Generated
+88 -16
View File
@@ -64,23 +64,46 @@ dependencies = [
name = "affine_common"
version = "0.1.0"
dependencies = [
"assert-json-diff",
"chrono",
"criterion",
"hex",
"nanoid",
"napi",
"pulldown-cmark",
"rand 0.9.4",
"rayon",
"serde",
"serde_json",
"sha3",
"tempfile",
]
[[package]]
name = "affine_doc_loader"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683b3f18f85e1c8528321903d602ca32a5603f4539ca1412c765f0dbaf6aa63f"
dependencies = [
"nanoid",
"pulldown-cmark",
"serde",
"serde_json",
"thiserror 2.0.18",
"y-octo",
]
[[package]]
name = "affine_importer"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1119144030a9ad25dee926d333dcb20433ab72f57425cbd159aa57eb9ee35119"
dependencies = [
"affine_doc_loader",
"chrono",
"nanoid",
"serde",
"serde_json",
"sha2 0.11.0",
"thiserror 2.0.18",
"zip",
]
[[package]]
name = "affine_media_capture"
version = "0.0.0"
@@ -138,6 +161,7 @@ name = "affine_native"
version = "0.0.0"
dependencies = [
"affine_common",
"affine_importer",
"affine_media_capture",
"affine_nbstore",
"affine_sqlite_v1",
@@ -156,6 +180,7 @@ dependencies = [
"typst-as-lib",
"typst-svg",
"uuid",
"zip",
]
[[package]]
@@ -163,6 +188,7 @@ name = "affine_nbstore"
version = "0.0.0"
dependencies = [
"affine_common",
"affine_doc_loader",
"affine_schema",
"anyhow",
"bincode 2.0.1",
@@ -196,6 +222,7 @@ version = "1.0.0"
dependencies = [
"aes-gcm",
"affine_common",
"affine_doc_loader",
"anyhow",
"assetpack-core",
"assetpack-transform-precomp2",
@@ -483,16 +510,6 @@ dependencies = [
"winnow 0.7.15",
]
[[package]]
name = "assert-json-diff"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "assetpack-core"
version = "0.1.0"
@@ -847,6 +864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
dependencies = [
"hybrid-array",
"zeroize",
]
[[package]]
@@ -1357,6 +1375,12 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
[[package]]
name = "constant_time_eq"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
[[package]]
name = "convert_case"
version = "0.11.0"
@@ -1850,6 +1874,12 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "deflate64"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2"
[[package]]
name = "der"
version = "0.7.10"
@@ -1914,6 +1944,7 @@ dependencies = [
"const-oid 0.10.2",
"crypto-common 0.2.2",
"ctutils",
"zeroize",
]
[[package]]
@@ -5080,6 +5111,16 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42"
[[package]]
name = "pbkdf2"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629"
dependencies = [
"digest 0.11.3",
"hmac 0.13.0",
]
[[package]]
name = "pdf_oxide"
version = "0.3.65"
@@ -5456,6 +5497,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppmd-rust"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
[[package]]
name = "ppv-lite86"
version = "0.2.21"
@@ -6551,6 +6598,17 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "sha1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
"digest 0.11.3",
]
[[package]]
name = "sha2"
version = "0.10.9"
@@ -6871,7 +6929,7 @@ dependencies = [
"rand 0.8.6",
"rsa",
"serde",
"sha1",
"sha1 0.10.6",
"sha2 0.10.9",
"smallvec",
"sqlx-core",
@@ -7537,6 +7595,7 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
dependencies = [
"deranged",
"itoa",
"js-sys",
"num-conv",
"powerfmt",
"serde_core",
@@ -9876,12 +9935,25 @@ version = "8.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
dependencies = [
"aes 0.9.1",
"bzip2",
"constant_time_eq",
"crc32fast",
"deflate64",
"flate2",
"getrandom 0.4.2",
"hmac 0.13.0",
"indexmap",
"lzma-rust2",
"memchr",
"pbkdf2",
"ppmd-rust",
"sha1 0.11.0",
"time",
"typed-path",
"zeroize",
"zopfli",
"zstd",
]
[[package]]
Cargo.toml
+3
View File
@@ -16,6 +16,8 @@ resolver = "3"
[workspace.dependencies]
aes-gcm = "0.10"
affine_common = { path = "./packages/common/native" }
affine_doc_loader = "0.1.0"
affine_importer = "0.1.0"
affine_nbstore = { path = "./packages/frontend/native/nbstore" }
anyhow = "1"
assert-json-diff = "2.0"
@@ -116,6 +118,7 @@ resolver = "3"
] }
windows-core = { version = "0.61" }
y-octo = "0.0.3"
zip = "8.6"
[profile.dev.package.sqlx-macros]
opt-level = 3
blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts
+325 -29
View File
@@ -2,7 +2,11 @@ import { readFileSync } from 'node:fs';
import { basename, resolve } from 'node:path';
import {
BearTransformer,
commitImportBatchToWorkspace,
type ImportBatch,
MarkdownTransformer,
NotionHtmlTransformer,
ObsidianTransformer,
} from '@blocksuite/affine/widgets/linked-doc';
import {
@@ -84,6 +88,23 @@ function zipFixture(entries: Record<string, string | Uint8Array>) {
return new Blob([buffer], { type: 'application/zip' });
}
async function commitPlannedImport<T extends { batch: ImportBatch }>(
collection: TestWorkspace,
schema: Schema,
planned: T
) {
const committed = await commitImportBatchToWorkspace(
collection,
schema,
planned.batch
);
return {
...planned,
...committed,
docIds: committed.docIds,
};
}
function exportSnapshot(doc: Store): DocSnapshot {
const job = doc.getTransformer([
docLinkBaseURLMiddleware(doc.workspace.id),
@@ -203,6 +224,12 @@ function snapshotDocByTitle(
return simplifyBlockForSnapshot(exportSnapshot(doc!).blocks, titleById);
}
function titleMap(collection: TestWorkspace) {
return new Map(
collection.meta.docMetas.map(meta => [meta.id, meta.title ?? '<untitled>'])
);
}
function collectSimplifiedDeltas(
block: Record<string, unknown>
): Record<string, unknown>[] {
@@ -218,6 +245,27 @@ function collectSimplifiedDeltas(
return [...deltas, ...childDeltas];
}
function collectSnapshotDeltas(
block: BlockSnapshot
): DeltaInsert<AffineTextAttributes>[] {
const text = block.props.text as
| { delta?: DeltaInsert<AffineTextAttributes>[] }
| undefined;
return [
...(text?.delta ?? []),
...(block.children ?? []).flatMap(child => collectSnapshotDeltas(child)),
];
}
function folderChild(
folder: { children: Map<string, unknown> } | undefined,
name: string
) {
return folder?.children.get(name) as
| { children: Map<string, unknown>; pageId?: string; icon?: unknown }
| undefined;
}
describe('snapshot to markdown', () => {
test('code', async () => {
const blockSnapshot: BlockSnapshot = {
@@ -377,13 +425,16 @@ Hello world
'# Nested Page\nNested body',
});
const { docIds, folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
const { docIds, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
});
})
);
expect(docIds).toHaveLength(2);
expect(
@@ -431,13 +482,16 @@ Hello world
'# SDK架构\nNested body',
});
const { folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
const { folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
});
})
);
const [rootFolder] = [...(folderHierarchy?.children.values() ?? [])];
expect(rootFolder?.name).toBe('Export');
@@ -463,12 +517,16 @@ Hello world
'---\ntitle: Frontmatter Title\n---\nBody',
});
const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({
const { docIds } = await commitPlannedImport(
collection,
schema,
imported,
extensions: testStoreExtensions,
});
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(1);
expect(collection.meta.getDocMeta(docIds[0])?.title).toBe(
@@ -491,12 +549,16 @@ Hello world
'test/2.md': 'target page',
});
const { docIds } = await MarkdownTransformer.importMarkdownZip({
const { docIds } = await commitPlannedImport(
collection,
schema,
imported,
extensions: testStoreExtensions,
});
await MarkdownTransformer.planMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(2);
const titleById = new Map(
@@ -527,6 +589,88 @@ Hello world
});
});
test('imports markdown zip assets, nested zip, CJK paths, and duplicate names', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const imported = zipFixture({
'入口.md':
'![logo](./assets/logo.png)\n[同名](./folder/duplicate.md)\n![archive](./nested.zip)',
'assets/logo.png': new Uint8Array([137, 80, 78, 71]),
'folder/duplicate.md': 'folder duplicate',
'other/duplicate.md': 'other duplicate',
'nested.zip': zipBytes({
'ignored.md': 'nested markdown should stay an attachment',
}),
});
const { docIds, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(3);
expect(
collection.meta.docMetas
.map(meta => meta.title)
.sort((a, b) => (a ?? '').localeCompare(b ?? ''))
).toEqual(['duplicate', 'duplicate', '入口']);
const titles = titleMap(collection);
const entry = snapshotDocByTitle(collection, '入口', titles);
expect(JSON.stringify(entry)).toContain('"sourceId":"<asset>"');
const entrySnapshot = exportSnapshot(
collection
.getDoc(
collection.meta.docMetas.find(meta => meta.title === '入口')!.id
)!
.getStore({
id: collection.meta.docMetas.find(meta => meta.title === '入口')!.id,
})
);
const linkedPageDelta = collectSnapshotDeltas(entrySnapshot.blocks).find(
delta => delta.attributes?.reference?.type === 'LinkedPage'
);
const linkedPageId =
linkedPageDelta?.attributes?.reference?.type === 'LinkedPage'
? linkedPageDelta.attributes.reference.pageId
: undefined;
expect(linkedPageId).toBeTruthy();
expect(
JSON.stringify(
snapshotDocByTitle(
collection,
'duplicate',
new Map([[linkedPageId!, 'duplicate']])
)
)
).toContain('folder duplicate');
expect(collectSimplifiedDeltas(entry)).toContainEqual({
insert: ' ',
reference: {
type: 'LinkedPage',
page: 'duplicate',
title: '同名',
},
});
expect(JSON.stringify(entry).match(/"sourceId":"<asset>"/g)).toHaveLength(
2
);
expect(folderHierarchy?.children.has('folder')).toBe(true);
expect(folderHierarchy?.children.has('other')).toBe(true);
expect(
collection.meta.docMetas.some(meta => meta.title === 'ignored')
).toBe(false);
});
test('imports notion markdown zip relative doc links as linked pages', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
@@ -540,12 +684,16 @@ Hello world
'# Target\ntarget page',
});
const { docIds } = await MarkdownTransformer.importNotionMarkdownZip({
const { docIds } = await commitPlannedImport(
collection,
schema,
imported,
extensions: testStoreExtensions,
});
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(2);
const titleById = new Map(
@@ -587,13 +735,16 @@ Hello world
}),
});
const { docIds, folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
const { docIds, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
});
})
);
expect(docIds).toHaveLength(4);
const titleById = new Map(
@@ -646,16 +797,20 @@ Hello world
'vault/archive.zip'
);
const { docIds } = await ObsidianTransformer.importObsidianVault({
const { docIds } = await commitPlannedImport(
collection,
schema,
importedFiles: [
markdownFixture('entry.md'),
markdownFixture('linked.md'),
attachment,
],
extensions: testStoreExtensions,
});
await ObsidianTransformer.planObsidianVault({
collection,
schema,
importedFiles: [
markdownFixture('entry.md'),
markdownFixture('linked.md'),
attachment,
],
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(2);
const titleById = new Map(
@@ -673,6 +828,147 @@ Hello world
}).toMatchSnapshot();
});
test('imports notion html zip golden baseline', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const imported = zipFixture({
'Export/index.html': '<html><body>workspace index</body></html>',
'Export/Project.html': `
<html>
<body>
<div class="page-header-icon undefined"><span class="icon">✅</span></div>
<div class="page-body">
<p id="11111111-1111-1111-1111-111111111111" class="">Project body</p>
<img id="22222222-2222-2222-2222-222222222222" src="assets/logo.png" />
</div>
</body>
</html>
`,
'Export/Project/Nested.html': `
<html>
<body>
<div class="page-body"><p id="33333333-3333-3333-3333-333333333333" class="">Nested body</p></div>
</body>
</html>
`,
'Export/assets/logo.png': new Uint8Array([137, 80, 78, 71]),
});
const result = await commitPlannedImport(
collection,
schema,
await NotionHtmlTransformer.planNotionHtmlZip({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(result.isWorkspaceFile).toBe(true);
expect(result.hasMarkdown).toBe(false);
expect(result.pageIds).toHaveLength(2);
expect(collection.meta.docMetas.map(meta => meta.title)).toEqual(['', '']);
const titles = titleMap(collection);
const importedSnapshots = result.pageIds.map(pageId =>
simplifyBlockForSnapshot(
exportSnapshot(collection.getDoc(pageId)!.getStore({ id: pageId }))
.blocks,
titles
)
);
const projectSnapshot = importedSnapshots.find(snapshot =>
JSON.stringify(snapshot).includes('Project body')
);
expect(projectSnapshot).toBeTruthy();
expect(JSON.stringify(projectSnapshot)).toContain('Project body');
expect(JSON.stringify(projectSnapshot)).toContain('"sourceId":"<asset>"');
const exportFolder = folderChild(result.folderHierarchy, 'Export');
const projectNode = folderChild(exportFolder, 'Project');
expect(result.pageIds).toContain(projectNode?.pageId);
expect(projectNode?.icon).toEqual({ type: 'emoji', content: '✅' });
expect(result.pageIds).toContain(
folderChild(projectNode, 'Nested')?.pageId
);
});
test('imports bear backup golden baseline', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const imported = zipFixture({
'Notes/Idea.textbundle/text.md': [
'# Bear Title',
'',
'![photo](assets/photo.png)',
'',
'==🟢green highlight==',
'',
'#work/project',
'#Blue Tag#',
].join('\n'),
'Notes/Idea.textbundle/info.json': JSON.stringify({
'net.shinyfrog.bear': {
creationDate: '2024-01-02T03:04:05.000Z',
modificationDate: '2024-01-03T03:04:05.000Z',
},
}),
'Notes/Idea.textbundle/assets/photo.png': new Uint8Array([
137, 80, 78, 71,
]),
});
const { docIds, tags, folderHierarchy } = await commitPlannedImport(
collection,
schema,
await BearTransformer.planBearBackup({
collection,
schema,
imported,
extensions: testStoreExtensions,
})
);
expect(docIds).toHaveLength(1);
const meta = collection.meta.getDocMeta(docIds[0]);
expect(meta?.title).toBe('Bear Title');
expect(meta?.createDate).toBe(Date.parse('2024-01-02T03:04:05.000Z'));
expect(meta?.updatedDate).toBe(Date.parse('2024-01-03T03:04:05.000Z'));
expect([...tags.keys()]).toEqual(['Blue Tag', 'work/project']);
const titles = titleMap(collection);
const snapshot = snapshotDocByTitle(collection, 'Bear Title', titles);
expect(JSON.stringify(snapshot)).toContain('"sourceId":"<asset>"');
expect(JSON.stringify(snapshot)).toContain('green highlight');
const blueTag = folderChild(folderHierarchy, 'Blue Tag');
expect([
...(
(blueTag?.children as Map<string, unknown> | undefined) ?? new Map()
).values(),
]).toEqual(
expect.arrayContaining([expect.objectContaining({ pageId: docIds[0] })])
);
const project = folderChild(
folderChild(folderHierarchy, 'work'),
'project'
);
expect([
...(
(project?.children as Map<string, unknown> | undefined) ?? new Map()
).values(),
]).toEqual(
expect.arrayContaining([expect.objectContaining({ pageId: docIds[0] })])
);
});
test('paragraph', async () => {
const blockSnapshot: BlockSnapshot = {
type: 'block',
blocksuite/affine/shared/src/utils/file/filesys.ts
+123 -18
View File
@@ -142,6 +142,55 @@ type AcceptTypes =
| 'Docx'
| 'MindMap';
type OpenFileOptions = {
fileSystemAccess?: boolean;
snapshot?: boolean;
};
function copyToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
function sleep(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms));
}
export async function snapshotFile(file: File, relativePath?: string) {
let bytes: Uint8Array | null = null;
for (let attempt = 0; attempt < 3; attempt++) {
try {
bytes = new Uint8Array(await file.arrayBuffer());
break;
} catch (error) {
if (!isNotReadableError(error) || attempt === 2) {
throw error;
}
await sleep(100 * (attempt + 1));
}
}
if (!bytes) {
throw new DOMException('File could not be read', 'NotReadableError');
}
const snapshot = new File([copyToArrayBuffer(bytes)], file.name, {
type: file.type,
lastModified: file.lastModified,
});
const path = relativePath ?? file.webkitRelativePath;
if (path) {
Object.defineProperty(snapshot, 'webkitRelativePath', {
value: path,
writable: false,
});
}
return snapshot;
}
export async function snapshotFiles(files: File[]) {
return Promise.all(files.map(file => snapshotFile(file)));
}
function canUseFileSystemAccessAPI(
api: 'showOpenFilePicker' | 'showDirectoryPicker'
) {
@@ -157,11 +206,21 @@ function canUseFileSystemAccessAPI(
);
}
function isAbortError(error: unknown) {
return error instanceof Error && error.name === 'AbortError';
}
function isNotReadableError(error: unknown) {
return error instanceof Error && error.name === 'NotReadableError';
}
export async function openFilesWith(
acceptType: AcceptTypes = 'Any',
multiple: boolean = true
multiple: boolean = true,
options?: OpenFileOptions
): Promise<File[] | null> {
const supportsFileSystemAccess =
options?.fileSystemAccess !== false &&
canUseFileSystemAccessAPI('showOpenFilePicker');
// If the File System Access API is supported…
@@ -180,15 +239,19 @@ export async function openFilesWith(
// Show the file picker, optionally allowing multiple files.
const handles = await window.showOpenFilePicker(pickerOpts);
return await Promise.all(handles.map(handle => handle.getFile()));
const files = await Promise.all(handles.map(handle => handle.getFile()));
return options?.snapshot ? await snapshotFiles(files) : files;
} catch (err) {
if (options?.snapshot && !isAbortError(err)) {
throw err;
}
console.error(err);
return null;
}
}
// Fallback if the File System Access API is not supported.
return new Promise(resolve => {
return new Promise((resolve, reject) => {
// Append a new `<input type="file" multiple? />` and hide it.
const input = document.createElement('input');
input.classList.add('affine-upload-input');
@@ -205,13 +268,27 @@ export async function openFilesWith(
document.body.append(input);
// The `change` event fires when the user interacts with the dialog.
input.addEventListener('change', () => {
// Remove the `<input type="file" multiple? />` again from the DOM.
input.remove();
const files = input.files ? Array.from(input.files) : null;
const finish = (result: File[] | null) => {
// Remove the `<input type="file" multiple? />` again from the DOM.
input.remove();
resolve(result);
};
resolve(input.files ? Array.from(input.files) : null);
if (files && options?.snapshot) {
snapshotFiles(files).then(finish, error => {
input.remove();
reject(error);
});
} else {
finish(files);
}
});
// The `cancel` event fires when the user cancels the dialog.
input.addEventListener('cancel', () => resolve(null));
input.addEventListener('cancel', () => {
input.remove();
resolve(null);
});
// Show the picker.
if ('showPicker' in HTMLInputElement.prototype) {
input.showPicker();
@@ -221,12 +298,18 @@ export async function openFilesWith(
});
}
export async function openDirectory(): Promise<File[] | null> {
export async function openDirectory(
options?: OpenFileOptions
): Promise<File[] | null> {
const supportsFileSystemAccess = canUseFileSystemAccessAPI(
'showDirectoryPicker'
);
if (supportsFileSystemAccess && window.showDirectoryPicker) {
if (
options?.fileSystemAccess !== false &&
supportsFileSystemAccess &&
window.showDirectoryPicker
) {
try {
const dirHandle = await window.showDirectoryPicker();
const files: File[] = [];
@@ -241,11 +324,15 @@ export async function openDirectory(): Promise<File[] | null> {
const fileHandle = handle as FileSystemFileHandle;
if (fileHandle.getFile) {
const file = await fileHandle.getFile();
Object.defineProperty(file, 'webkitRelativePath', {
value: relativePath,
writable: false,
});
files.push(file);
if (options?.snapshot) {
files.push(await snapshotFile(file, relativePath));
} else {
Object.defineProperty(file, 'webkitRelativePath', {
value: relativePath,
writable: false,
});
files.push(file);
}
}
} else if (handle.kind === 'directory') {
await readDirectory(
@@ -259,12 +346,15 @@ export async function openDirectory(): Promise<File[] | null> {
await readDirectory(dirHandle, '');
return files;
} catch (err) {
if (options?.snapshot && !isAbortError(err)) {
throw err;
}
console.error(err);
return null;
}
}
return new Promise(resolve => {
return new Promise((resolve, reject) => {
const input = document.createElement('input');
input.classList.add('affine-upload-input');
input.style.display = 'none';
@@ -276,11 +366,26 @@ export async function openDirectory(): Promise<File[] | null> {
document.body.append(input);
input.addEventListener('change', () => {
input.remove();
resolve(input.files ? Array.from(input.files) : null);
const files = input.files ? Array.from(input.files) : null;
const finish = (result: File[] | null) => {
input.remove();
resolve(result);
};
if (files && options?.snapshot) {
snapshotFiles(files).then(finish, error => {
input.remove();
reject(error);
});
} else {
finish(files);
}
});
input.addEventListener('cancel', () => resolve(null));
input.addEventListener('cancel', () => {
input.remove();
resolve(null);
});
if ('showPicker' in HTMLInputElement.prototype) {
input.showPicker();
blocksuite/affine/widgets/linked-doc/src/import-doc/import-doc.ts
+59 -144
View File
@@ -2,15 +2,9 @@ import {
CloseIcon,
ExportToHTMLIcon,
ExportToMarkdownIcon,
HelpIcon,
NewIcon,
NotionIcon,
} from '@blocksuite/affine-components/icons';
import {
openDirectory,
openFilesWith,
openSingleFileWith,
} from '@blocksuite/affine-shared/utils';
import { openFilesWith } from '@blocksuite/affine-shared/utils';
import { WithDisposable } from '@blocksuite/global/lit';
import type { ExtensionType, Schema, Workspace } from '@blocksuite/store';
import { html, LitElement, type PropertyValues } from 'lit';
@@ -18,8 +12,6 @@ import { query, state } from 'lit/decorators.js';
import { HtmlTransformer } from '../transformers/html.js';
import { MarkdownTransformer } from '../transformers/markdown.js';
import { NotionHtmlTransformer } from '../transformers/notion-html.js';
import { ObsidianTransformer } from '../transformers/obsidian.js';
import { styles } from './styles.js';
export type OnSuccessHandler = (
@@ -59,114 +51,69 @@ export class ImportDoc extends WithDisposable(LitElement) {
}
private async _importHtml() {
const files = await openFilesWith('Html');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const needLoading = file.size > SHOW_LOADING_SIZE;
const fileName = file.name.split('.').slice(0, -1).join('.');
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await HtmlTransformer.importHTMLToDoc({
collection: this.collection,
schema: this.schema,
extensions: this.extensions,
html: text,
fileName,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
try {
const files = await openFilesWith('Html');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const needLoading = file.size > SHOW_LOADING_SIZE;
const fileName = file.name.split('.').slice(0, -1).join('.');
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await HtmlTransformer.importHTMLToDoc({
collection: this.collection,
schema: this.schema,
extensions: this.extensions,
html: text,
fileName,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
}
}
this._onImportSuccess(pageIds);
} catch (error) {
this._onFail(error);
}
this._onImportSuccess(pageIds);
}
private async _importMarkDown() {
const files = await openFilesWith('Markdown');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const fileName = file.name.split('.').slice(0, -1).join('.');
const needLoading = file.size > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await MarkdownTransformer.importMarkdownToDoc({
collection: this.collection,
schema: this.schema,
markdown: text,
fileName,
extensions: this.extensions,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
try {
const files = await openFilesWith('Markdown');
if (!files) return;
const pageIds: string[] = [];
for (const file of files) {
const text = await file.text();
const fileName = file.name.split('.').slice(0, -1).join('.');
const needLoading = file.size > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const pageId = await MarkdownTransformer.importMarkdownToDoc({
collection: this.collection,
schema: this.schema,
markdown: text,
fileName,
extensions: this.extensions,
});
needLoading && this.abortController.abort();
if (pageId) {
pageIds.push(pageId);
}
}
this._onImportSuccess(pageIds);
} catch (error) {
this._onFail(error);
}
this._onImportSuccess(pageIds);
}
private async _importNotion() {
const file = await openSingleFileWith('Zip');
if (!file) return;
const needLoading = file.size > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const { entryId, pageIds, isWorkspaceFile, hasMarkdown } =
await NotionHtmlTransformer.importNotionZip({
collection: this.collection,
schema: this.schema,
imported: file,
extensions: this.extensions,
});
needLoading && this.abortController.abort();
if (hasMarkdown) {
this._onFail(
'Importing markdown files from Notion is deprecated. Please export your Notion pages as HTML.'
);
return;
}
this._onImportSuccess(entryId ? [entryId] : [], {
isWorkspaceFile,
importedCount: pageIds.length,
});
}
private async _importObsidian() {
const files = await openDirectory();
if (!files || files.length === 0) return;
const needLoading =
files.reduce((acc, f) => acc + f.size, 0) > SHOW_LOADING_SIZE;
if (needLoading) {
this.hidden = false;
this._loading = true;
} else {
this.abortController.abort();
}
const { docIds, docEmojis } = await ObsidianTransformer.importObsidianVault(
{
collection: this.collection,
schema: this.schema,
importedFiles: files,
extensions: this.extensions,
}
);
needLoading && this.abortController.abort();
this._onImportSuccess(docIds, { docEmojis });
}
private _onCloseClick(event: MouseEvent) {
@@ -174,8 +121,8 @@ export class ImportDoc extends WithDisposable(LitElement) {
this.abortController.abort();
}
private _onFail(message: string) {
this.onFail?.(message);
private _onFail(error: unknown) {
this.onFail?.(error instanceof Error ? error.message : String(error));
}
private _onImportSuccess(
@@ -213,14 +160,6 @@ export class ImportDoc extends WithDisposable(LitElement) {
window.removeEventListener('mousemove', this._onMouseMove);
}
private _openLearnImportLink(event: MouseEvent) {
event.stopPropagation();
window.open(
'https://affine.pro/blog/import-your-data-from-notion-into-affine',
'_blank'
);
}
override render() {
if (this._loading) {
return html`
@@ -276,30 +215,6 @@ export class ImportDoc extends WithDisposable(LitElement) {
</icon-button>
</div>
<div class="button-container">
<icon-button
class="button-item"
text="Notion"
@click="${this._importNotion}"
>
${NotionIcon}
<div
slot="suffix"
class="button-suffix"
@click="${this._openLearnImportLink}"
>
${HelpIcon}
<affine-tooltip>
Learn how to Import your Notion pages into AFFiNE.
</affine-tooltip>
</div>
</icon-button>
<icon-button
class="button-item"
text="Obsidian"
@click="${this._importObsidian}"
>
${ExportToMarkdownIcon}
</icon-button>
<icon-button class="button-item" text="Coming soon..." disabled>
${NewIcon}
</icon-button>
blocksuite/affine/widgets/linked-doc/src/transformers/bear.ts
+122 -40
View File
@@ -7,17 +7,30 @@ import {
} from '@blocksuite/affine-shared/adapters';
import { Container } from '@blocksuite/global/di';
import { sha } from '@blocksuite/global/utils';
import type { ExtensionType, Schema, Workspace } from '@blocksuite/store';
import type {
DocSnapshot,
ExtensionType,
Schema,
Workspace,
} from '@blocksuite/store';
import { extMimeMap, Transformer } from '@blocksuite/store';
import JSZip from 'jszip';
import {
blobsFromAssets,
type ImportBatch,
type ImportDoc,
type ImportFolder,
type ImportTag,
type ImportWarning,
} from './import-batch.js';
import { createCollectionDocCRUD } from './markdown.js';
/** Recursive tree node representing a tag-based folder hierarchy. */
type FolderHierarchy = {
export type BearFolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
children: Map<string, BearFolderHierarchy>;
pageId?: string;
parentPath?: string;
};
@@ -29,10 +42,11 @@ type BearImportOptions = {
extensions: ExtensionType[];
};
type BearImportResult = {
export type PlanBearBackupResult = {
docIds: string[];
tags: Map<string, string[]>;
folderHierarchy: FolderHierarchy;
folderHierarchy: BearFolderHierarchy;
batch: ImportBatch;
};
type BundleEntry = {
@@ -182,8 +196,8 @@ function deduplicateTags(tags: string[]): string[] {
*/
function buildFolderHierarchyFromTags(
tagDocMap: Map<string, string[]>
): FolderHierarchy {
const root: FolderHierarchy = {
): BearFolderHierarchy {
const root: BearFolderHierarchy = {
name: '',
path: '',
children: new Map(),
@@ -226,6 +240,34 @@ function buildFolderHierarchyFromTags(
return root;
}
function flattenFolderHierarchy(root: BearFolderHierarchy): ImportFolder[] {
const folders: ImportFolder[] = [];
const visit = (node: BearFolderHierarchy) => {
if (node.name) {
folders.push({
path: node.path,
name: node.name,
parentPath: node.parentPath,
pageId: node.pageId,
});
}
for (const child of node.children.values()) {
visit(child);
}
};
for (const child of root.children.values()) {
visit(child);
}
return folders;
}
function tagsToBatchTags(tags: Map<string, string[]>): ImportTag[] {
return Array.from(tags, ([name, docIds]) => ({ name, docIds }));
}
const GFM_CALLOUT_MAP: Record<string, string> = {
IMPORTANT: '\u26A0',
NOTE: '\uD83D\uDCDD',
@@ -250,9 +292,9 @@ function convertGfmCallouts(markdown: string): string {
if (!inCodeBlock) {
lines[i] = lines[i].replace(
/^(>\s*)\[!(\w+)\]/,
(_match, prefix: string, type: string) => {
(match, prefix: string, type: string) => {
const emoji = GFM_CALLOUT_MAP[type.toUpperCase()];
return emoji ? `${prefix}[!${emoji}]` : _match;
return emoji ? `${prefix}[!${emoji}]` : match;
}
);
}
@@ -306,7 +348,7 @@ function convertHighlights(markdown: string): string {
if (!inCodeBlock) {
lines[i] = lines[i].replace(
/==(\S(?:[^=]|=[^=])*?)==/g,
(_match, content: string) => {
(...[, content]) => {
const firstChar = String.fromCodePoint(content.codePointAt(0)!);
const color = HIGHLIGHT_COLOR_MAP[firstChar];
if (color) {
@@ -340,16 +382,24 @@ function extractTitle(markdown: string, bundleName: string): string {
return bundleName.replace(/\.textbundle$/i, '') || 'Untitled';
}
function applySnapshotTitle(snapshot: DocSnapshot, title: string) {
snapshot.meta.title = title;
snapshot.blocks.props.title = {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
};
}
/**
* Import a Bear .bear2bk backup file.
* Uses JSZip for lazy/streaming decompression to handle large backups.
*/
async function importBearBackup({
async function planBearBackup({
collection,
schema,
imported,
extensions,
}: BearImportOptions): Promise<BearImportResult> {
}: BearImportOptions): Promise<PlanBearBackupResult> {
const provider = getProvider(extensions);
// JSZip reads the zip directory without decompressing all entries
@@ -358,7 +408,7 @@ async function importBearBackup({
// Scan entries and group by textbundle
const bundleMap = new Map<string, BundleEntry>();
zip.forEach((path, _entry) => {
zip.forEach(path => {
if (path.includes('__MACOSX') || path.includes('.DS_Store')) return;
const tbMatch = path.match(/^(.+?\.textbundle)\/(.*)/i);
@@ -421,6 +471,12 @@ async function importBearBackup({
}
const docIds: string[] = [];
const docs: ImportDoc[] = [];
const importBlobs = new Map<
string,
Awaited<ReturnType<typeof blobsFromAssets>>[0]
>();
const warnings: ImportWarning[] = [];
const tagDocMap = new Map<string, string[]>();
// Process bundles sequentially to limit memory.
@@ -497,44 +553,70 @@ async function importBearBackup({
}
const mdAdapter = new MarkdownAdapter(job, provider);
const doc = await mdAdapter.toDoc({
const snapshot = await mdAdapter.toDocSnapshot({
file: markdown,
assets: job.assetsManager,
});
if (doc) {
docIds.push(doc.id);
const metaPatch: Record<string, unknown> = {};
if (bearMeta?.creationDate) {
const ts = Date.parse(String(bearMeta.creationDate));
if (!isNaN(ts)) metaPatch.createDate = ts;
}
if (bearMeta?.modificationDate) {
const ts = Date.parse(String(bearMeta.modificationDate));
if (!isNaN(ts)) metaPatch.updatedDate = ts;
}
if (Object.keys(metaPatch).length) {
collection.meta.setDocMeta(doc.id, metaPatch);
}
for (const tag of tags) {
if (!tagDocMap.has(tag)) {
tagDocMap.set(tag, []);
}
tagDocMap.get(tag)!.push(doc.id);
}
const docId = collection.idGenerator();
snapshot.meta.id = docId;
applySnapshotTitle(snapshot, title);
const meta: ImportDoc['meta'] = { title };
if (bearMeta?.creationDate) {
const ts = Date.parse(String(bearMeta.creationDate));
if (!isNaN(ts)) meta.createDate = ts;
}
} catch (err) {
console.warn(`Failed to import bundle: ${entry.bundlePath}`, err);
if (bearMeta?.modificationDate) {
const ts = Date.parse(String(bearMeta.modificationDate));
if (!isNaN(ts)) meta.updatedDate = ts;
}
docs.push({
id: docId,
snapshot,
meta,
});
docIds.push(docId);
for (const tag of tags) {
if (!tagDocMap.has(tag)) {
tagDocMap.set(tag, []);
}
tagDocMap.get(tag)!.push(docId);
}
for (const blob of await blobsFromAssets(
pendingAssets,
pendingPathBlobIdMap
)) {
importBlobs.set(blob.blobId, blob);
}
} catch {
warnings.push({
code: 'bear-bundle-import-failed',
message: `Failed to import bundle: ${entry.bundlePath}`,
sourcePath: entry.bundlePath,
});
}
}
const folderHierarchy = buildFolderHierarchyFromTags(tagDocMap);
return { docIds, tags: tagDocMap, folderHierarchy };
return {
docIds,
tags: tagDocMap,
folderHierarchy,
batch: {
docs,
blobs: Array.from(importBlobs.values()),
folders: flattenFolderHierarchy(folderHierarchy),
tags: tagsToBatchTags(tagDocMap),
warnings,
done: true,
},
};
}
/** Public API for importing Bear .bear2bk backup archives. */
export const BearTransformer = {
importBearBackup,
planBearBackup,
};
blocksuite/affine/widgets/linked-doc/src/transformers/import-batch.ts
+157
View File
@@ -0,0 +1,157 @@
import {
type DocMeta,
type DocSnapshot,
type Schema,
Transformer,
type Workspace,
} from '@blocksuite/store';
export type ImportBlob = {
blobId: string;
sourcePath: string;
fileName: string;
mime: string;
bytes: Uint8Array;
};
export type ImportIconData =
| {
type: 'emoji';
unicode: string;
}
| {
type: 'image';
content: string;
};
export type ImportFolder = {
path: string;
name: string;
parentPath?: string;
pageId?: string;
icon?: ImportIconData;
};
export type ImportDoc = {
id: string;
sourcePath?: string;
snapshot: DocSnapshot;
meta?: Partial<
Pick<
DocMeta,
'title' | 'createDate' | 'updatedDate' | 'tags' | 'favorite' | 'trash'
>
>;
};
export type ImportTag = {
name: string;
docIds: string[];
};
export type ImportIcon = {
docId: string;
icon: ImportIconData;
};
export type ImportWarning = {
code: string;
message: string;
sourcePath?: string;
};
export type ImportBatch = {
docs: ImportDoc[];
blobs: ImportBlob[];
folders?: ImportFolder[];
tags?: ImportTag[];
icons?: ImportIcon[];
warnings?: ImportWarning[];
progress?: {
completed: number;
total: number;
};
entryId?: string;
isWorkspaceFile?: boolean;
done: boolean;
};
export type ImportCommitResult = {
docIds: string[];
entryId?: string;
isWorkspaceFile?: boolean;
rootFolderId?: string;
warnings: ImportWarning[];
};
export async function blobsFromAssets(
assets: ReadonlyMap<string, File>,
pathBlobIdMap: ReadonlyMap<string, string> = new Map()
) {
const sourcePathByBlobId = new Map<string, string>();
for (const [path, blobId] of pathBlobIdMap) {
sourcePathByBlobId.set(blobId, path);
}
return Promise.all(
Array.from(
assets,
async ([blobId, file]): Promise<ImportBlob> => ({
blobId,
sourcePath: sourcePathByBlobId.get(blobId) ?? file.name,
fileName: file.name,
mime: file.type,
bytes: new Uint8Array(await file.arrayBuffer()),
})
)
);
}
function copyToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
export async function commitImportBatchToWorkspace(
collection: Workspace,
schema: Schema,
batch: ImportBatch
): Promise<ImportCommitResult> {
for (const blob of batch.blobs) {
await collection.blobSync.set(
blob.blobId,
new File([copyToArrayBuffer(blob.bytes)], blob.fileName, {
type: blob.mime,
})
);
}
const transformer = new Transformer({
schema,
blobCRUD: collection.blobSync,
docCRUD: {
create: (id: string) => collection.createDoc(id).getStore({ id }),
get: (id: string) => collection.getDoc(id)?.getStore({ id }) ?? null,
delete: (id: string) => collection.removeDoc(id),
},
middlewares: [],
});
const docIds: string[] = [];
for (const doc of batch.docs) {
const store = await transformer.snapshotToDoc(doc.snapshot);
if (!store) continue;
docIds.push(store.id);
if (doc.meta && Object.keys(doc.meta).length) {
collection.meta.setDocMeta(store.id, doc.meta);
}
}
return {
docIds,
entryId: batch.entryId,
isWorkspaceFile: batch.isWorkspaceFile,
warnings: batch.warnings ?? [],
};
}
blocksuite/affine/widgets/linked-doc/src/transformers/index.ts
+14 -1
View File
@@ -1,9 +1,22 @@
export { BearTransformer } from './bear.js';
export { DocxTransformer } from './docx.js';
export { HtmlTransformer } from './html.js';
export {
blobsFromAssets,
commitImportBatchToWorkspace,
type ImportBatch,
type ImportBlob,
type ImportCommitResult,
type ImportDoc,
type ImportFolder,
type ImportIcon,
type ImportIconData,
type ImportTag,
type ImportWarning,
} from './import-batch.js';
export { MarkdownTransformer } from './markdown.js';
export { NotionHtmlTransformer } from './notion-html.js';
export { ObsidianTransformer } from './obsidian.js';
export { PdfTransformer } from './pdf.js';
export { createAssetsArchive, download } from './utils.js';
export { createAssetsArchive, download, Unzip } from './utils.js';
export { ZipTransformer } from './zip.js';
blocksuite/affine/widgets/linked-doc/src/transformers/markdown.ts
+76 -20
View File
@@ -16,6 +16,7 @@ import { BlockSuiteError, ErrorCode } from '@blocksuite/global/exceptions';
import { sha } from '@blocksuite/global/utils';
import type {
DocMeta,
DocSnapshot,
ExtensionType,
Schema,
Store,
@@ -23,6 +24,12 @@ import type {
} from '@blocksuite/store';
import { extMimeMap, Transformer } from '@blocksuite/store';
import {
blobsFromAssets,
type ImportBatch,
type ImportDoc,
type ImportFolder,
} from './import-batch.js';
import type { AssetMap, ImportedFileEntry, PathBlobIdMap } from './type.js';
import { createAssetsArchive, download, parseMatter, Unzip } from './utils.js';
@@ -438,6 +445,14 @@ function prepareNotionMarkdownFile({
};
}
function applySnapshotTitle(snapshot: DocSnapshot, title: string) {
snapshot.meta.title = title;
snapshot.blocks.props.title = {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
};
}
/**
* Filters hidden/system entries that should never participate in imports.
*/
@@ -685,7 +700,7 @@ async function importMarkdownToDoc({
* @param options.imported The zip file as a Blob
* @returns A Promise that resolves to an array of IDs of the newly created docs
*/
type FolderHierarchy = {
export type FolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
@@ -693,18 +708,19 @@ type FolderHierarchy = {
parentPath?: string;
};
export type ImportMarkdownZipResult = {
export type PlanMarkdownZipResult = {
docIds: string[];
folderHierarchy?: FolderHierarchy;
batch: ImportBatch;
};
async function importMarkdownZip({
async function planMarkdownZip({
collection,
schema,
imported,
extensions,
}: ImportMarkdownZipOptions): Promise<ImportMarkdownZipResult> {
return importMarkdownZipInternal({
}: ImportMarkdownZipOptions): Promise<PlanMarkdownZipResult> {
return planMarkdownZipInternal({
collection,
schema,
imported,
@@ -712,13 +728,13 @@ async function importMarkdownZip({
});
}
async function importNotionMarkdownZip({
async function planNotionMarkdownZip({
collection,
schema,
imported,
extensions,
}: ImportMarkdownZipOptions): Promise<ImportMarkdownZipResult> {
return importMarkdownZipInternal({
}: ImportMarkdownZipOptions): Promise<PlanMarkdownZipResult> {
return planMarkdownZipInternal({
collection,
schema,
imported,
@@ -731,7 +747,7 @@ async function importNotionMarkdownZip({
});
}
async function importMarkdownZipInternal({
async function planMarkdownZipInternal({
collection,
schema,
imported,
@@ -741,12 +757,13 @@ async function importMarkdownZipInternal({
prepareMarkdownFile = prepareDefaultMarkdownFile,
preserveCommonRoot = false,
recursiveZip = false,
}: ImportMarkdownZipInternalOptions): Promise<ImportMarkdownZipResult> {
}: ImportMarkdownZipInternalOptions): Promise<PlanMarkdownZipResult> {
const provider = getProvider([
markdownZipDocLinkToDeltaMatcher,
...extensions,
]);
const docIds: string[] = [];
const docs: ImportDoc[] = [];
const pendingAssets: AssetMap = new Map();
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
const docPathMap: Array<{ fullPath: string; docId: string }> = [];
@@ -812,16 +829,20 @@ async function importMarkdownZipInternal({
assets: job.assetsManager,
});
snapshot.meta.id = pageId;
const doc = await job.snapshotToDoc(snapshot);
if (doc) {
applyMetaPatch(collection, doc.id, meta);
docIds.push(doc.id);
docPathMap.push({ fullPath, docId: doc.id });
}
applySnapshotTitle(snapshot, preferredTitle);
docs.push({
id: pageId,
snapshot,
meta: {
...meta,
title: preferredTitle,
},
});
docIds.push(pageId);
docPathMap.push({ fullPath, docId: pageId });
})
);
// Build folder hierarchy from zip paths
const folderHierarchy = buildMarkdownZipFolderHierarchy(
docPathMap,
normalizeFolderName,
@@ -829,7 +850,18 @@ async function importMarkdownZipInternal({
createRootFolderForTopLevelDocs
);
return { docIds, folderHierarchy };
return {
docIds,
folderHierarchy,
batch: {
docs,
blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap),
folders: folderHierarchy
? flattenFolderHierarchy(folderHierarchy)
: undefined,
done: true,
},
};
}
/**
@@ -921,10 +953,34 @@ function buildMarkdownZipFolderHierarchy(
return root.children.size > 0 ? root : undefined;
}
function flattenFolderHierarchy(root: FolderHierarchy): ImportFolder[] {
const folders: ImportFolder[] = [];
const visit = (node: FolderHierarchy) => {
if (node.name) {
folders.push({
path: node.path,
name: node.name,
parentPath: node.parentPath,
pageId: node.pageId,
});
}
for (const child of node.children.values()) {
visit(child);
}
};
for (const child of root.children.values()) {
visit(child);
}
return folders;
}
export const MarkdownTransformer = {
exportDoc,
importMarkdownToBlock,
importMarkdownToDoc,
importMarkdownZip,
importNotionMarkdownZip,
planMarkdownZip,
planNotionMarkdownZip,
};
blocksuite/affine/widgets/linked-doc/src/transformers/notion-html.ts
+101 -93
View File
@@ -12,6 +12,14 @@ import {
type Workspace,
} from '@blocksuite/store';
import {
blobsFromAssets,
type ImportBatch,
type ImportDoc,
type ImportFolder,
type ImportIconData,
type ImportWarning,
} from './import-batch.js';
import { Unzip } from './utils.js';
type ImportNotionZipOptions = {
@@ -21,12 +29,12 @@ type ImportNotionZipOptions = {
extensions: ExtensionType[];
};
type PageIcon = {
export type PageIcon = {
type: 'emoji' | 'image';
content: string; // emoji unicode or image URL/data
content: string;
};
type FolderHierarchy = {
export type FolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
@@ -35,12 +43,13 @@ type FolderHierarchy = {
icon?: PageIcon;
};
type ImportNotionZipResult = {
export type PlanNotionHtmlZipResult = {
entryId: string | undefined;
pageIds: string[];
isWorkspaceFile: boolean;
hasMarkdown: boolean;
folderHierarchy?: FolderHierarchy;
batch: ImportBatch;
};
function getProvider(extensions: ExtensionType[]) {
@@ -61,22 +70,9 @@ function parseFolderPath(filePath: string): {
}
function extractPageIcon(doc: Document): PageIcon | undefined {
// Look for Notion page icon in the HTML
// Notion export format: <div class="page-header-icon undefined"><span class="icon">✅</span></div>
console.log('=== Extracting page icon ===');
// Check if there's a head section with title for debugging
const headTitle = doc.querySelector('head title');
if (headTitle) {
console.log('Page title from head:', headTitle.textContent);
}
// Look for the exact Notion export structure: .page-header-icon .icon
const notionIconSpan = doc.querySelector('.page-header-icon .icon');
if (notionIconSpan && notionIconSpan.textContent) {
const iconContent = notionIconSpan.textContent.trim();
console.log('Found Notion icon (.page-header-icon .icon):', iconContent);
if (/\p{Emoji}/u.test(iconContent)) {
return {
type: 'emoji',
@@ -85,50 +81,29 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
// Look for page header area for debugging
const pageHeader = doc.querySelector('.page-header-icon');
if (pageHeader) {
console.log(
'Found .page-header-icon:',
pageHeader.outerHTML.substring(0, 300) + '...'
);
}
// Fallback: try to find emoji icons with older selectors
const emojiIcon = doc.querySelector('.page-header-icon .notion-emoji');
if (emojiIcon && emojiIcon.textContent) {
console.log(
'Found emoji icon (.page-header-icon .notion-emoji):',
emojiIcon.textContent
);
return {
type: 'emoji',
content: emojiIcon.textContent.trim(),
};
}
// Try alternative emoji selectors
const altEmojiIcon = doc.querySelector('[role="img"][aria-label]');
if (
altEmojiIcon &&
altEmojiIcon.textContent &&
/\p{Emoji}/u.test(altEmojiIcon.textContent)
) {
console.log(
'Found emoji icon ([role="img"][aria-label]):',
altEmojiIcon.textContent
);
return {
type: 'emoji',
content: altEmojiIcon.textContent.trim(),
};
}
// Look for image icons in the page header
const imageIcon = doc.querySelector('.page-header-icon img');
if (imageIcon) {
const src = imageIcon.getAttribute('src');
console.log('Found image icon (.page-header-icon img):', src);
if (src) {
return {
type: 'image',
@@ -137,27 +112,15 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
// Fallback: Look for any span with emoji class "icon" in page header area
const iconSpans = doc.querySelectorAll('span.icon');
for (const span of iconSpans) {
if (span.textContent && /\p{Emoji}/u.test(span.textContent.trim())) {
const parent = span.parentElement;
console.log(
'Found emoji in span.icon:',
span.textContent,
'parent classes:',
parent?.className
);
// Check if this is in a page header context
if (
parent &&
(parent.classList.contains('page-header-icon') ||
parent.closest('.page-header-icon'))
) {
console.log(
'Using emoji from span.icon in page header:',
span.textContent
);
return {
type: 'emoji',
content: span.textContent.trim(),
@@ -166,15 +129,11 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
// Fallback: Try to find icons in the page title area that might contain emoji
const pageTitle = doc.querySelector('.page-title, h1');
if (pageTitle && pageTitle.textContent) {
console.log('Page title element found:', pageTitle.textContent);
const text = pageTitle.textContent.trim();
// Check if the title starts with an emoji
const emojiMatch = text.match(/^(\p{Emoji}+)/u);
if (emojiMatch) {
console.log('Found emoji in title:', emojiMatch[1]);
return {
type: 'emoji',
content: emojiMatch[1],
@@ -182,7 +141,6 @@ function extractPageIcon(doc: Document): PageIcon | undefined {
}
}
console.log('No page icon found');
return undefined;
}
@@ -200,7 +158,6 @@ function buildFolderHierarchy(
let current = root;
let currentPath = '';
// Navigate/create folder structure
for (const folderName of folderParts) {
const parentPath = currentPath;
currentPath = currentPath ? `${currentPath}/${folderName}` : folderName;
@@ -216,7 +173,6 @@ function buildFolderHierarchy(
current = current.children.get(folderName)!;
}
// If this is a page file, associate it with the current folder
if (fileName.endsWith('.html') && !fileName.startsWith('index.html')) {
const pageName = fileName.replace(/\.html$/, '');
if (!current.children.has(pageName)) {
@@ -229,7 +185,6 @@ function buildFolderHierarchy(
icon: icon,
});
} else {
// Update existing entry with pageId and icon
const existingPage = current.children.get(pageName)!;
existingPage.pageId = pageId;
if (icon) {
@@ -242,29 +197,59 @@ function buildFolderHierarchy(
return root;
}
/**
* Imports a Notion zip file into the BlockSuite collection.
*
* @param options - The options for importing.
* @param options.collection - The BlockSuite document collection.
* @param options.schema - The schema of the BlockSuite document collection.
* @param options.imported - The imported zip file as a Blob.
*
* @returns A promise that resolves to an object containing:
* - entryId: The ID of the entry page (if any).
* - pageIds: An array of imported page IDs.
* - isWorkspaceFile: Whether the imported file is a workspace file.
* - hasMarkdown: Whether the zip contains markdown files.
* - folderHierarchy: The parsed folder hierarchy from the Notion export.
*/
async function importNotionZip({
function toImportIconData(icon?: PageIcon): ImportIconData | undefined {
if (!icon) return undefined;
if (icon.type === 'emoji') {
return {
type: 'emoji',
unicode: icon.content,
};
}
return {
type: 'image',
content: icon.content,
};
}
function flattenFolderHierarchy(root: FolderHierarchy): ImportFolder[] {
const folders: ImportFolder[] = [];
const visit = (node: FolderHierarchy) => {
if (node.name) {
folders.push({
path: node.path,
name: node.name,
parentPath: node.parentPath,
pageId: node.pageId,
icon: toImportIconData(node.icon),
});
}
for (const child of node.children.values()) {
visit(child);
}
};
for (const child of root.children.values()) {
visit(child);
}
return folders;
}
async function planNotionHtmlZip({
collection,
schema,
imported,
extensions,
}: ImportNotionZipOptions): Promise<ImportNotionZipResult> {
}: ImportNotionZipOptions): Promise<PlanNotionHtmlZipResult> {
const provider = getProvider(extensions);
const pageIds: string[] = [];
const docs: ImportDoc[] = [];
const blobs = new Map<
string,
Awaited<ReturnType<typeof blobsFromAssets>>[0]
>();
const warnings: ImportWarning[] = [];
let isWorkspaceFile = false;
let hasMarkdown = false;
let entryId: string | undefined;
@@ -280,7 +265,7 @@ async function importNotionZip({
const pageMap = new Map<string, string>();
const pagePaths: string[] = [];
const promises: Promise<void>[] = [];
const pendingAssets = new Map<string, Blob>();
const pendingAssets = new Map<string, File>();
const pendingPathBlobIdMap = new Map<string, string>();
for (const { path, content, index } of unzip) {
if (path.startsWith('__MACOSX/')) continue;
@@ -305,11 +290,7 @@ async function importNotionZip({
const text = await content.text();
const doc = new DOMParser().parseFromString(text, 'text/html');
const pageBody = doc.querySelector('.page-body');
if (pageBody && pageBody.children.length === 0) {
// Skip empty pages
continue;
}
// Extract page icon from the HTML
if (pageBody && pageBody.children.length === 0) continue;
pageIcon = extractPageIcon(doc);
}
@@ -327,10 +308,12 @@ async function importNotionZip({
continue;
}
if (index === 0 && fileName.endsWith('.csv')) {
window.open(
'https://affine.pro/blog/import-your-data-from-notion-into-affine',
'_blank'
);
warnings.push({
code: 'notion-csv-export',
message:
'The imported Notion export appears to be CSV instead of HTML.',
sourcePath: path,
});
continue;
}
if (fileName.endsWith('.zip')) {
@@ -342,7 +325,7 @@ async function importNotionZip({
}
const blob = content;
const ext = path.split('.').at(-1) ?? '';
const mime = extMimeMap.get(ext) ?? '';
const mime = extMimeMap.get(ext.toLowerCase()) ?? '';
const key = await sha(await blob.arrayBuffer());
const filePathSplit = path.split('/');
while (filePathSplit.length > 1) {
@@ -375,32 +358,57 @@ async function importNotionZip({
pathBlobIdMap.set(key, value);
}
}
const page = await htmlAdapter.toDoc({
const snapshot = await htmlAdapter.toDocSnapshot({
file: await zipFile.get(path)!.text(),
pageId: pageMap.get(path),
pageMap,
assets: job.assetsManager,
});
if (page) {
pageIds.push(page.id);
}
docs.push({
id: snapshot.meta.id,
snapshot,
});
pageIds.push(snapshot.meta.id);
});
promises.push(...pagePromises);
promises.push(
blobsFromAssets(pendingAssets, pendingPathBlobIdMap).then(importBlobs => {
for (const blob of importBlobs) {
blobs.set(blob.blobId, blob);
}
})
);
return promises;
};
const allPromises = await parseZipFile(imported);
await Promise.all(allPromises.flat());
entryId = entryId ?? pageIds[0];
// Build folder hierarchy from collected paths
const folderHierarchy =
pagePathsWithIds.length > 0
? buildFolderHierarchy(pagePathsWithIds)
: undefined;
return { entryId, pageIds, isWorkspaceFile, hasMarkdown, folderHierarchy };
return {
entryId,
pageIds,
isWorkspaceFile,
hasMarkdown,
folderHierarchy,
batch: {
docs,
blobs: Array.from(blobs.values()),
folders: folderHierarchy
? flattenFolderHierarchy(folderHierarchy)
: undefined,
warnings,
entryId,
isWorkspaceFile,
done: true,
},
};
}
export const NotionHtmlTransformer = {
importNotionZip,
planNotionHtmlZip,
};
blocksuite/affine/widgets/linked-doc/src/transformers/obsidian.ts
+34 -16
View File
@@ -20,7 +20,11 @@ import { extMimeMap, nanoid } from '@blocksuite/store';
import type { Html, Text } from 'mdast';
import {
applyMetaPatch,
blobsFromAssets,
type ImportBatch,
type ImportDoc,
} from './import-batch.js';
import {
bindImportedAssetsToJob,
createMarkdownImportJob,
getProvider,
@@ -129,8 +133,8 @@ function preprocessTitleHeader(markdown: string): string {
function preprocessObsidianCallouts(markdown: string): string {
return markdown.replace(
/^(> *)\[!([^\]\n]+)\]([+-]?)([^\n]*)/gm,
(_, prefix, type, _fold, rest) => {
/^(> *)\[!([^\]\n]+)\](?:[+-]?)([^\n]*)/gm,
(_, prefix, type, rest) => {
const calloutToken =
CALLOUT_TYPE_MAP[type.trim().toLowerCase()] ?? DEFAULT_CALLOUT_EMOJI;
const title = rest.trim();
@@ -688,12 +692,16 @@ export type ImportObsidianVaultResult = {
docEmojis: Map<string, string>;
};
export async function importObsidianVault({
export type PlanObsidianVaultResult = ImportObsidianVaultResult & {
batch: ImportBatch;
};
export async function planObsidianVault({
collection,
schema,
importedFiles,
extensions,
}: ImportObsidianVaultOptions): Promise<ImportObsidianVaultResult> {
}: ImportObsidianVaultOptions): Promise<PlanObsidianVaultResult> {
const provider = getProvider([
obsidianWikilinkToDeltaMatcher,
obsidianAttachmentEmbedMarkdownAdapterMatcher,
@@ -701,6 +709,7 @@ export async function importObsidianVault({
]);
const docIds: string[] = [];
const docs: ImportDoc[] = [];
const docEmojis = new Map<string, string>();
const pendingAssets: AssetMap = new Map();
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
@@ -813,22 +822,31 @@ export async function importObsidianVault({
if (snapshot) {
snapshot.meta.id = predefinedId;
const doc = await job.snapshotToDoc(snapshot);
if (doc) {
applyMetaPatch(collection, doc.id, {
...meta,
title: preferredTitle,
trash: false,
});
docIds.push(doc.id);
}
docs.push({
id: predefinedId,
snapshot,
meta: { ...meta, title: preferredTitle, trash: false },
});
docIds.push(predefinedId);
}
})
);
return { docIds, docEmojis };
return {
docIds,
docEmojis,
batch: {
docs,
blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap),
icons: Array.from(docEmojis, ([docId, emoji]) => ({
docId,
icon: { type: 'emoji', unicode: emoji },
})),
done: true,
},
};
}
export const ObsidianTransformer = {
importObsidianVault,
planObsidianVault,
};
blocksuite/playground/apps/_common/components/starter-debug-menu.ts
+14 -3
View File
@@ -54,6 +54,7 @@ import {
type Workspace,
} from '@blocksuite/affine/store';
import {
commitImportBatchToWorkspace,
createAssetsArchive,
download,
HtmlTransformer,
@@ -436,12 +437,17 @@ export class StarterDebugMenu extends ShadowlessElement {
try {
const file = await openSingleFileWith('Zip');
if (!file) return;
const { docIds } = await MarkdownTransformer.importMarkdownZip({
const { batch } = await MarkdownTransformer.planMarkdownZip({
collection: this.collection,
schema: this.editor.doc.schema,
imported: file,
extensions: this._getStoreManager().get('store'),
});
const { docIds } = await commitImportBatchToWorkspace(
this.collection,
this.editor.doc.schema,
batch
);
if (!this.editor.host) return;
toast(
this.editor.host,
@@ -473,16 +479,21 @@ export class StarterDebugMenu extends ShadowlessElement {
try {
const file = await openSingleFileWith('Zip');
if (!file) return;
const result = await NotionHtmlTransformer.importNotionZip({
const { batch } = await NotionHtmlTransformer.planNotionHtmlZip({
collection: this.collection,
schema: this.editor.doc.schema,
imported: file,
extensions: this._getStoreManager().get('store'),
});
const result = await commitImportBatchToWorkspace(
this.collection,
this.editor.doc.schema,
batch
);
if (!this.editor.host) return;
toast(
this.editor.host,
`Successfully imported ${result.pageIds.length} Notion HTML pages.`
`Successfully imported ${result.docIds.length} Notion HTML pages.`
);
} catch (error) {
console.error('Failed to import Notion HTML Zip:', error);
packages/backend/native/Cargo.toml
+2 -5
View File
@@ -10,11 +10,8 @@ crate-type = ["cdylib"]
[dependencies]
aes-gcm = { workspace = true }
affine_common = { workspace = true, features = [
"hashcash",
"napi",
"ydoc-loader",
] }
affine_common = { workspace = true, features = ["hashcash", "napi"] }
affine_doc_loader = { workspace = true }
anyhow = { workspace = true }
assetpack-core = "0.1.0"
assetpack-transform-precomp2 = "0.1.0"
packages/backend/native/src/doc.rs
+15 -15
View File
@@ -1,6 +1,6 @@
use affine_common::{
doc_parser::{self, BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent},
napi_utils::map_napi_err,
use affine_common::napi_utils::map_napi_err;
use affine_doc_loader::{
self as doc_loader, BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent,
};
use napi::bindgen_prelude::*;
use napi_derive::napi;
@@ -109,7 +109,7 @@ impl From<CrawlResult> for NativeCrawlResult {
#[napi]
pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result<NativeCrawlResult> {
let result = map_napi_err(
doc_parser::parse_doc_from_binary(doc_bin.into(), doc_id),
doc_loader::parse_doc_from_binary(doc_bin.into(), doc_id),
Status::GenericFailure,
)?;
Ok(result.into())
@@ -118,7 +118,7 @@ pub fn parse_doc_from_binary(doc_bin: Buffer, doc_id: String) -> Result<NativeCr
#[napi]
pub fn parse_page_doc(doc_bin: Buffer, max_summary_length: Option<i32>) -> Result<Option<NativePageDocContent>> {
let result = map_napi_err(
doc_parser::parse_page_doc(doc_bin.into(), max_summary_length.map(|v| v as isize)),
doc_loader::parse_page_doc(doc_bin.into(), max_summary_length.map(|v| v as isize)),
Status::GenericFailure,
)?;
Ok(result.map(Into::into))
@@ -126,7 +126,7 @@ pub fn parse_page_doc(doc_bin: Buffer, max_summary_length: Option<i32>) -> Resul
#[napi]
pub fn parse_workspace_doc(doc_bin: Buffer) -> Result<Option<NativeWorkspaceDocContent>> {
let result = map_napi_err(doc_parser::parse_workspace_doc(doc_bin.into()), Status::GenericFailure)?;
let result = map_napi_err(doc_loader::parse_workspace_doc(doc_bin.into()), Status::GenericFailure)?;
Ok(result.map(Into::into))
}
@@ -138,7 +138,7 @@ pub fn parse_doc_to_markdown(
doc_url_prefix: Option<String>,
) -> Result<NativeMarkdownResult> {
let result = map_napi_err(
doc_parser::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false), doc_url_prefix),
doc_loader::parse_doc_to_markdown(doc_bin.into(), doc_id, ai_editable.unwrap_or(false), doc_url_prefix),
Status::GenericFailure,
)?;
Ok(result.into())
@@ -147,7 +147,7 @@ pub fn parse_doc_to_markdown(
#[napi]
pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option<bool>) -> Result<Vec<String>> {
let result = map_napi_err(
doc_parser::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false)),
doc_loader::get_doc_ids_from_binary(doc_bin.into(), include_trash.unwrap_or(false)),
Status::GenericFailure,
)?;
Ok(result)
@@ -165,7 +165,7 @@ pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option<boo
#[napi]
pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::build_full_doc(&title, &markdown, &doc_id),
doc_loader::build_full_doc(&title, &markdown, &doc_id),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -184,7 +184,7 @@ pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String)
#[napi]
pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, doc_id: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_doc(&existing_binary, &new_markdown, &doc_id),
doc_loader::update_doc(&existing_binary, &new_markdown, &doc_id),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -202,7 +202,7 @@ pub fn update_doc_with_markdown(existing_binary: Buffer, new_markdown: String, d
#[napi]
pub fn update_doc_title(existing_binary: Buffer, title: String, doc_id: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_doc_title(&existing_binary, &doc_id, &title),
doc_loader::update_doc_title(&existing_binary, &doc_id, &title),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -229,7 +229,7 @@ pub fn update_doc_properties(
updated_by: Option<String>,
) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_doc_properties(
doc_loader::update_doc_properties(
&existing_binary,
&properties_doc_id,
&target_doc_id,
@@ -254,7 +254,7 @@ pub fn update_doc_properties(
#[napi]
pub fn add_doc_to_root_doc(root_doc_bin: Buffer, doc_id: String, title: Option<String>) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::add_doc_to_root_doc(root_doc_bin.into(), &doc_id, title.as_deref()),
doc_loader::add_doc_to_root_doc(root_doc_bin.into(), &doc_id, title.as_deref()),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -267,7 +267,7 @@ pub fn build_public_root_doc(root_doc_bin: Buffer, doc_metas: Vec<PublicDocMetaI
.map(|meta| (meta.id.as_str(), meta.title.as_deref()))
.collect::<Vec<_>>();
let result = map_napi_err(
doc_parser::build_public_root_doc(&root_doc_bin, &metas),
doc_loader::build_public_root_doc(&root_doc_bin, &metas),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
@@ -285,7 +285,7 @@ pub fn build_public_root_doc(root_doc_bin: Buffer, doc_metas: Vec<PublicDocMetaI
#[napi]
pub fn update_root_doc_meta_title(root_doc_bin: Buffer, doc_id: String, title: String) -> Result<Buffer> {
let result = map_napi_err(
doc_parser::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title),
doc_loader::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title),
Status::GenericFailure,
)?;
Ok(Buffer::from(result))
packages/backend/native/src/runtime/storage_runtime/doc_blob_refs.rs
+4 -4
View File
@@ -1,4 +1,4 @@
use affine_common::doc_parser;
use affine_doc_loader as doc_loader;
use chrono::{DateTime, Utc};
use sqlx::{FromRow, PgPool};
use y_octo::Doc;
@@ -102,7 +102,7 @@ async fn load_workspace_doc_ids(pool: &PgPool, workspace_id: &str) -> RuntimeRes
let Some(root) = load_current_doc(pool, workspace_id, workspace_id).await? else {
return Ok(Vec::new());
};
let ids = doc_parser::get_doc_ids_from_binary(root.blob, false)
let ids = doc_loader::get_doc_ids_from_binary(root.blob, false)
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs root doc parse failed: {err}")))?;
let mut ids = ids;
ids.sort();
@@ -206,7 +206,7 @@ async fn purge_removed_doc_refs(pool: &PgPool, workspace_id: &str, current_doc_i
}
fn extract_refs(snapshot: &SnapshotRow) -> RuntimeResult<Vec<ExtractedRef>> {
let parsed = doc_parser::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone())
let parsed = doc_loader::parse_doc_from_binary(snapshot.blob.clone(), snapshot.doc_id.clone())
.map_err(|err| RuntimeError::invalid_state(format!("Doc blob refs parse failed: {err}")))?;
let mut refs = Vec::new();
for block in parsed.blocks {
@@ -232,7 +232,7 @@ mod tests {
fn doc_blob_refs_extracts_image_refs() {
let doc_id = "doc-blob-ref-test".to_string();
let blob =
doc_parser::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build");
doc_loader::build_full_doc("Doc", "![Alt](blob://image-blob-key)", &doc_id).expect("doc fixture should build");
let snapshot = SnapshotRow {
workspace_id: "workspace".to_string(),
doc_id,
packages/backend/server/src/schema.gql
+1 -1
View File
@@ -2959,4 +2959,4 @@ type tokenType {
refresh: String!
sessionToken: String
token: String!
}
}
packages/common/native/Cargo.toml
+7 -23
View File
@@ -6,32 +6,16 @@ publish = false
version = "0.1.0"
[features]
default = []
default = []
hashcash = ["chrono", "hex", "sha3", "rand"]
napi = ["dep:napi"]
ydoc-loader = [
"assert-json-diff",
"nanoid",
"pulldown-cmark",
"serde",
"serde_json",
"thiserror",
"y-octo",
]
napi = ["dep:napi"]
[dependencies]
assert-json-diff = { workspace = true, optional = true }
chrono = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
nanoid = { workspace = true, optional = true }
napi = { workspace = true, optional = true }
pulldown-cmark = { workspace = true, optional = true }
rand = { workspace = true, optional = true }
serde = { workspace = true, optional = true, features = ["derive"] }
serde_json = { workspace = true, optional = true }
sha3 = { workspace = true, optional = true }
thiserror = { workspace = true, optional = true }
y-octo = { workspace = true, optional = true }
chrono = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
napi = { workspace = true, optional = true }
rand = { workspace = true, optional = true }
sha3 = { workspace = true, optional = true }
[dev-dependencies]
criterion = { workspace = true }
packages/common/native/src/doc_parser/block_spec.rs
-601
View File
@@ -1,601 +0,0 @@
use y_octo::{Any, Map, TextAttributes, TextDeltaOp, TextInsert};
use super::{
ParseError,
blocksuite::get_string,
schema::{
PROP_CAPTION, PROP_CHECKED, PROP_COLUMN_ID_SUFFIX, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
PROP_ORDER_SUFFIX, PROP_ROW_ID_SUFFIX, PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL,
PROP_VIDEO_ID, PROP_WIDTH, SYS_FLAVOUR, table_cell_text_key,
},
table::{MarkdownTableOptions, render_markdown_table},
value::{value_to_f64, value_to_string},
};
/// Block flavours used in AFFiNE documents.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlockFlavour {
Paragraph,
List,
Code,
Divider,
Image,
Table,
Bookmark,
EmbedYoutube,
EmbedIframe,
Callout,
}
impl BlockFlavour {
pub fn as_str(&self) -> &'static str {
match self {
BlockFlavour::Paragraph => "affine:paragraph",
BlockFlavour::List => "affine:list",
BlockFlavour::Code => "affine:code",
BlockFlavour::Divider => "affine:divider",
BlockFlavour::Image => "affine:image",
BlockFlavour::Table => "affine:table",
BlockFlavour::Bookmark => "affine:bookmark",
BlockFlavour::EmbedYoutube => "affine:embed-youtube",
BlockFlavour::EmbedIframe => "affine:embed-iframe",
BlockFlavour::Callout => "affine:callout",
}
}
pub fn from_str(value: &str) -> Option<Self> {
match value {
"affine:paragraph" => Some(BlockFlavour::Paragraph),
"affine:list" => Some(BlockFlavour::List),
"affine:code" => Some(BlockFlavour::Code),
"affine:divider" => Some(BlockFlavour::Divider),
"affine:image" => Some(BlockFlavour::Image),
"affine:table" => Some(BlockFlavour::Table),
"affine:bookmark" => Some(BlockFlavour::Bookmark),
"affine:embed-youtube" => Some(BlockFlavour::EmbedYoutube),
"affine:embed-iframe" => Some(BlockFlavour::EmbedIframe),
"affine:callout" => Some(BlockFlavour::Callout),
_ => None,
}
}
}
/// Block types for paragraphs and lists.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BlockType {
// Paragraph types
Text,
H1,
H2,
H3,
H4,
H5,
H6,
Quote,
// List types
Bulleted,
Numbered,
Todo,
// Preserve unknown types when loading from ydoc.
Unknown(String),
}
impl BlockType {
pub fn as_str(&self) -> &str {
match self {
BlockType::Text => "text",
BlockType::H1 => "h1",
BlockType::H2 => "h2",
BlockType::H3 => "h3",
BlockType::H4 => "h4",
BlockType::H5 => "h5",
BlockType::H6 => "h6",
BlockType::Quote => "quote",
BlockType::Bulleted => "bulleted",
BlockType::Numbered => "numbered",
BlockType::Todo => "todo",
BlockType::Unknown(value) => value.as_str(),
}
}
pub fn from_str(value: &str) -> Option<Self> {
match value {
"text" => Some(BlockType::Text),
"h1" => Some(BlockType::H1),
"h2" => Some(BlockType::H2),
"h3" => Some(BlockType::H3),
"h4" => Some(BlockType::H4),
"h5" => Some(BlockType::H5),
"h6" => Some(BlockType::H6),
"quote" => Some(BlockType::Quote),
"bulleted" => Some(BlockType::Bulleted),
"numbered" => Some(BlockType::Numbered),
"todo" => Some(BlockType::Todo),
_ => None,
}
}
pub fn from_str_lossy(value: String) -> Self {
Self::from_str(&value).unwrap_or(BlockType::Unknown(value))
}
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct ImageSpec {
pub(super) source_id: String,
pub(super) caption: Option<String>,
pub(super) width: Option<f64>,
pub(super) height: Option<f64>,
}
impl ImageSpec {
pub(super) fn render_markdown(&self) -> String {
let blob_url = format!("blob://{}", self.source_id);
let caption = self.caption.as_deref().unwrap_or("");
if self.width.is_some() || self.height.is_some() || !caption.is_empty() {
let width_text = self
.width
.map(|value| value.to_string())
.unwrap_or_else(|| "auto".into());
let height_text = self
.height
.map(|value| value.to_string())
.unwrap_or_else(|| "auto".into());
return format!(
"<img\n src=\"{blob_url}\"\n alt=\"{caption}\"\n width=\"{width_text}\"\n height=\"{height_text}\"\n/>\n\n"
);
}
let alt = if caption.is_empty() {
self.source_id.as_str()
} else {
caption
};
format!("\n![{alt}]({blob_url})\n\n")
}
pub(super) fn from_block_map(block: &Map) -> Self {
let source_id = get_string(block, PROP_SOURCE_ID).unwrap_or_default();
let caption = get_string(block, PROP_CAPTION);
let width = block.get(PROP_WIDTH).and_then(value_to_f64);
let height = block.get(PROP_HEIGHT).and_then(value_to_f64);
ImageSpec {
source_id,
caption,
width,
height,
}
}
pub(super) fn normalize_source(src: &str) -> Result<String, ParseError> {
let trimmed = src.trim();
if trimmed.is_empty() {
return Err(ParseError::ParserError("invalid_image_source".into()));
}
if let Some(rest) = trimmed.strip_prefix("blob://") {
if rest.is_empty() {
return Err(ParseError::ParserError("invalid_image_source".into()));
}
return Ok(rest.to_string());
}
if !trimmed.contains('/') && !trimmed.contains(':') {
return Ok(trimmed.to_string());
}
if let Some(pos) = trimmed.rfind("/blobs/") {
let id = &trimmed[pos + "/blobs/".len()..];
let id = id.split(['?', '#']).next().unwrap_or("");
if !id.is_empty() {
return Ok(id.to_string());
}
}
Err(ParseError::ParserError("unsupported_image_source".into()))
}
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct TableSpec {
pub(super) rows: Vec<Vec<String>>,
}
impl TableSpec {
pub(super) fn from_block_map(block: &Map) -> Self {
let mut row_entries: Vec<(String, String)> = Vec::new();
let mut column_entries: Vec<(String, String)> = Vec::new();
for key in block.keys() {
if key.starts_with(PROP_ROWS_PREFIX)
&& key.ends_with(PROP_ROW_ID_SUFFIX)
&& let Some(row_id) = block.get(key).and_then(|value| value_to_string(&value))
{
let base = key.trim_end_matches(PROP_ROW_ID_SUFFIX);
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
let order = block
.get(&order_key)
.and_then(|value| value_to_string(&value))
.unwrap_or_default();
row_entries.push((order, row_id));
}
if key.starts_with(PROP_COLUMNS_PREFIX)
&& key.ends_with(PROP_COLUMN_ID_SUFFIX)
&& let Some(column_id) = block.get(key).and_then(|value| value_to_string(&value))
{
let base = key.trim_end_matches(PROP_COLUMN_ID_SUFFIX);
let order_key = format!("{base}{PROP_ORDER_SUFFIX}");
let order = block
.get(&order_key)
.and_then(|value| value_to_string(&value))
.unwrap_or_default();
column_entries.push((order, column_id));
}
}
row_entries.sort_by(|a, b| a.0.cmp(&b.0));
column_entries.sort_by(|a, b| a.0.cmp(&b.0));
let mut rows = Vec::new();
for (_, row_id) in row_entries {
let mut row = Vec::new();
for (_, column_id) in &column_entries {
let cell_key = table_cell_text_key(&row_id, column_id);
let cell_text = block
.get(&cell_key)
.and_then(|value| value_to_string(&value))
.unwrap_or_default();
row.push(cell_text);
}
rows.push(row);
}
Self { rows }
}
pub(super) fn render_markdown(&self) -> Option<String> {
let options = MarkdownTableOptions::new(false, "<br />", true);
render_markdown_table(&self.rows, options)
}
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct BookmarkSpec {
pub(super) url: String,
pub(super) caption: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct EmbedYoutubeSpec {
pub(super) video_id: String,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct EmbedIframeSpec {
pub(super) url: String,
}
#[derive(Debug, Clone)]
pub(super) struct BlockSpec {
pub(super) flavour: BlockFlavour,
pub(super) block_type: Option<BlockType>,
pub(super) text: Vec<TextDeltaOp>,
pub(super) checked: Option<bool>,
pub(super) language: Option<String>,
pub(super) order: Option<i64>,
pub(super) image: Option<ImageSpec>,
pub(super) table: Option<TableSpec>,
pub(super) bookmark: Option<BookmarkSpec>,
pub(super) embed_youtube: Option<EmbedYoutubeSpec>,
pub(super) embed_iframe: Option<EmbedIframeSpec>,
}
impl BlockSpec {
pub(super) fn is_exact(&self, other: &BlockSpec) -> bool {
self.flavour == other.flavour
&& self.block_type == other.block_type
&& self.checked == other.checked
&& self.language == other.language
&& self.image == other.image
&& self.table == other.table
&& self.bookmark == other.bookmark
&& self.embed_youtube == other.embed_youtube
&& self.embed_iframe == other.embed_iframe
&& text_delta_eq(&self.text, &other.text)
}
pub(super) fn is_similar(&self, other: &BlockSpec) -> bool {
self.flavour == other.flavour && self.block_type == other.block_type
}
pub(super) fn block_type_str(&self) -> Option<&str> {
self.block_type.as_ref().map(BlockType::as_str)
}
pub(super) fn from_block_map(block: &Map) -> Result<Self, ParseError> {
let flavour_value = get_string(block, SYS_FLAVOUR).unwrap_or_default();
let Some(flavour) = BlockFlavour::from_str(&flavour_value) else {
return Err(ParseError::ParserError(format!(
"unsupported block flavour: {flavour_value}"
)));
};
Ok(Self::from_block_map_with_flavour(block, flavour))
}
pub(super) fn from_block_map_with_flavour(block: &Map, flavour: BlockFlavour) -> Self {
if flavour == BlockFlavour::Image {
return BlockSpec {
flavour,
block_type: None,
text: Vec::new(),
checked: None,
language: None,
order: None,
image: Some(ImageSpec::from_block_map(block)),
table: None,
bookmark: None,
embed_youtube: None,
embed_iframe: None,
};
}
if flavour == BlockFlavour::Table {
return BlockSpec {
flavour,
block_type: None,
text: Vec::new(),
checked: None,
language: None,
order: None,
image: None,
table: Some(TableSpec::from_block_map(block)),
bookmark: None,
embed_youtube: None,
embed_iframe: None,
};
}
if flavour == BlockFlavour::Bookmark {
return BlockSpec {
flavour,
block_type: None,
text: Vec::new(),
checked: None,
language: None,
order: None,
image: None,
table: None,
bookmark: Some(BookmarkSpec {
url: get_string(block, PROP_URL).unwrap_or_default(),
caption: get_string(block, PROP_CAPTION),
}),
embed_youtube: None,
embed_iframe: None,
};
}
if flavour == BlockFlavour::EmbedYoutube {
return BlockSpec {
flavour,
block_type: None,
text: Vec::new(),
checked: None,
language: None,
order: None,
image: None,
table: None,
bookmark: None,
embed_youtube: Some(EmbedYoutubeSpec {
video_id: get_string(block, PROP_VIDEO_ID).unwrap_or_default(),
}),
embed_iframe: None,
};
}
if flavour == BlockFlavour::EmbedIframe {
return BlockSpec {
flavour,
block_type: None,
text: Vec::new(),
checked: None,
language: None,
order: None,
image: None,
table: None,
bookmark: None,
embed_youtube: None,
embed_iframe: Some(EmbedIframeSpec {
url: get_string(block, PROP_URL).unwrap_or_default(),
}),
};
}
let block_type = match get_string(block, PROP_TYPE) {
Some(value) => Some(BlockType::from_str_lossy(value)),
None => match flavour {
BlockFlavour::Paragraph => Some(BlockType::Text),
BlockFlavour::List => Some(BlockType::Bulleted),
_ => None,
},
};
let text = block
.get(PROP_TEXT)
.and_then(|v| v.to_text())
.map(|text| text.to_delta())
.unwrap_or_default();
let checked = block.get(PROP_CHECKED).and_then(|v| v.to_any()).and_then(|a| match a {
Any::True => Some(true),
Any::False => Some(false),
_ => None,
});
let language = get_string(block, PROP_LANGUAGE);
let order = block.get(PROP_ORDER).and_then(|v| v.to_any()).and_then(|a| match a {
Any::Integer(value) => Some(value as i64),
Any::BigInt64(value) => Some(value),
Any::Float32(value) => Some(value.0 as i64),
Any::Float64(value) => Some(value.0 as i64),
_ => None,
});
BlockSpec {
flavour,
block_type,
text,
checked,
language,
order,
image: None,
table: None,
bookmark: None,
embed_youtube: None,
embed_iframe: None,
}
}
}
#[derive(Debug, Clone)]
pub(super) struct BlockNode {
pub(super) spec: BlockSpec,
pub(super) children: Vec<BlockNode>,
}
pub(super) trait TreeNode {
fn children(&self) -> &[Self]
where
Self: Sized;
}
impl TreeNode for BlockNode {
fn children(&self) -> &[BlockNode] {
&self.children
}
}
pub(super) fn count_tree_nodes<T: TreeNode>(nodes: &[T]) -> usize {
nodes.iter().map(|node| 1 + count_tree_nodes(node.children())).sum()
}
pub(super) fn text_delta_eq(a: &[TextDeltaOp], b: &[TextDeltaOp]) -> bool {
if a.len() != b.len() {
return false;
}
for (left, right) in a.iter().zip(b.iter()) {
match (left, right) {
(
TextDeltaOp::Insert {
insert: TextInsert::Text(text_a),
format: format_a,
},
TextDeltaOp::Insert {
insert: TextInsert::Text(text_b),
format: format_b,
},
) => {
if text_a != text_b {
return false;
}
if !attrs_eq(format_a.as_ref(), format_b.as_ref()) {
return false;
}
}
_ => return false,
}
}
true
}
fn attrs_eq(a: Option<&TextAttributes>, b: Option<&TextAttributes>) -> bool {
match (a, b) {
(None, None) => true,
(Some(left), Some(right)) => {
if left.len() != right.len() {
return false;
}
left.iter().all(|(key, value)| right.get(key) == Some(value))
}
_ => false,
}
}
#[cfg(test)]
mod tests {
use y_octo::DocOptions;
use super::{super::write::builder::text_ops_from_plain, *};
use crate::doc_parser::build_full_doc;
fn spec_from_markdown(markdown: &str, doc_id: &str, flavour: &str) -> BlockSpec {
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&bin).expect("apply update");
let blocks_map = doc.get_map("blocks").expect("blocks map");
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, SYS_FLAVOUR).as_deref() == Some(flavour)
{
return BlockSpec::from_block_map(&block_map).expect("spec");
}
}
panic!("block not found: {flavour}");
}
#[test]
fn test_from_block_map_paragraph() {
let spec = spec_from_markdown("Plain paragraph.", "block-spec-paragraph", "affine:paragraph");
assert_eq!(spec.flavour, BlockFlavour::Paragraph);
assert_eq!(spec.block_type, Some(BlockType::Text));
assert_eq!(spec.text, text_ops_from_plain("Plain paragraph."));
}
#[test]
fn test_from_block_map_list_checked() {
let spec = spec_from_markdown("- [x] Done", "block-spec-list", "affine:list");
assert_eq!(spec.flavour, BlockFlavour::List);
assert_eq!(spec.block_type, Some(BlockType::Todo));
assert_eq!(spec.checked, Some(true));
assert_eq!(spec.text, text_ops_from_plain("Done"));
}
#[test]
fn test_from_block_map_image() {
let spec = spec_from_markdown("![Alt](blob://image-id)", "block-spec-image", "affine:image");
assert_eq!(spec.flavour, BlockFlavour::Image);
let image = spec.image.expect("image spec");
assert_eq!(image.source_id, "image-id");
assert_eq!(image.caption.as_deref(), Some("Alt"));
assert_eq!(image.width, None);
assert_eq!(image.height, None);
}
#[test]
fn test_from_block_map_table() {
let spec = spec_from_markdown(
"| A | B |\n| --- | --- |\n| 1 | 2 |",
"block-spec-table",
"affine:table",
);
assert_eq!(spec.flavour, BlockFlavour::Table);
let table = spec.table.expect("table spec");
assert_eq!(
table.rows,
vec![
vec!["A".to_string(), "B".to_string()],
vec!["1".to_string(), "2".to_string()]
]
);
}
#[test]
fn test_from_block_map_embed_iframe() {
let spec = spec_from_markdown(
r#"<iframe src="https://example.com/embed"></iframe>"#,
"block-spec-embed-iframe",
"affine:embed-iframe",
);
assert_eq!(spec.flavour, BlockFlavour::EmbedIframe);
assert_eq!(spec.embed_iframe.as_ref().unwrap().url, "https://example.com/embed");
}
}
packages/common/native/src/doc_parser/blocksuite.rs
-176
View File
@@ -1,176 +0,0 @@
use std::collections::{HashMap, HashSet};
use y_octo::Map;
use super::{
schema::{SYS_CHILDREN, SYS_FLAVOUR, SYS_ID},
value::value_to_string,
};
pub(super) struct BlockIndex {
pub(super) block_pool: HashMap<String, Map>,
pub(super) parent_lookup: HashMap<String, String>,
}
pub(super) struct DocContext {
pub(super) block_pool: HashMap<String, Map>,
pub(super) parent_lookup: HashMap<String, String>,
pub(super) root_block_id: String,
}
pub(super) struct BlockWalker {
queue: Vec<(Option<String>, String)>,
visited: HashSet<String>,
}
pub(super) fn build_block_index(blocks_map: &Map) -> BlockIndex {
let mut block_pool: HashMap<String, Map> = HashMap::new();
let mut parent_lookup: HashMap<String, String> = HashMap::new();
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& let Some(block_id) = get_block_id(&block_map)
{
for child_id in collect_child_ids(&block_map) {
parent_lookup.insert(child_id, block_id.clone());
}
block_pool.insert(block_id, block_map);
}
}
BlockIndex {
block_pool,
parent_lookup,
}
}
impl DocContext {
pub(super) fn from_blocks_map(blocks_map: &Map, root_flavour: &str) -> Option<Self> {
let BlockIndex {
block_pool,
parent_lookup,
} = build_block_index(blocks_map);
let root_block_id = find_block_id_by_flavour(&block_pool, root_flavour)?;
Some(Self {
block_pool,
parent_lookup,
root_block_id,
})
}
pub(super) fn walker(&self) -> BlockWalker {
BlockWalker::new(&self.root_block_id)
}
}
impl BlockWalker {
fn new(root_block_id: &str) -> Self {
let mut visited = HashSet::new();
visited.insert(root_block_id.to_string());
Self {
queue: vec![(None, root_block_id.to_string())],
visited,
}
}
pub(super) fn next(&mut self) -> Option<(Option<String>, String)> {
self.queue.pop()
}
pub(super) fn enqueue_children(&mut self, parent_block_id: &str, block: &Map) {
let mut child_ids = collect_child_ids(block);
for child_id in child_ids.drain(..).rev() {
if self.visited.insert(child_id.clone()) {
self.queue.push((Some(parent_block_id.to_string()), child_id));
}
}
}
}
pub(super) fn find_block_id_by_flavour(block_pool: &HashMap<String, Map>, flavour: &str) -> Option<String> {
block_pool.iter().find_map(|(id, block)| {
get_flavour(block)
.filter(|block_flavour| block_flavour == flavour)
.map(|_| id.clone())
})
}
pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
block
.get(SYS_CHILDREN)
.and_then(|value| value.to_array())
.map(|array| {
array
.iter()
.filter_map(|value| value_to_string(&value))
.collect::<Vec<_>>()
})
.unwrap_or_default()
}
pub(super) fn get_block_id(block: &Map) -> Option<String> {
get_string(block, SYS_ID)
}
pub(super) fn get_flavour(block: &Map) -> Option<String> {
get_string(block, SYS_FLAVOUR)
}
pub(super) fn get_string(block: &Map, key: &str) -> Option<String> {
block.get(key).and_then(|value| value_to_string(&value))
}
pub(super) fn get_list_depth(
block_id: &str,
parent_lookup: &HashMap<String, String>,
blocks: &HashMap<String, Map>,
) -> usize {
let mut depth = 0;
let mut current_id = block_id.to_string();
while let Some(parent_id) = parent_lookup.get(&current_id) {
if let Some(parent_block) = blocks.get(parent_id)
&& get_flavour(parent_block).as_deref() == Some("affine:list")
{
depth += 1;
current_id = parent_id.clone();
continue;
}
break;
}
depth
}
pub(super) fn nearest_by_flavour(
start: &str,
flavour: &str,
parent_lookup: &HashMap<String, String>,
blocks: &HashMap<String, Map>,
) -> Option<Map> {
let mut cursor = Some(start.to_string());
while let Some(node) = cursor {
if let Some(block) = blocks.get(&node)
&& get_flavour(block).as_deref() == Some(flavour)
{
return Some(block.clone());
}
cursor = parent_lookup.get(&node).cloned();
}
None
}
pub(super) fn find_child_id_by_flavour(
parent: &Map,
block_pool: &HashMap<String, Map>,
flavour: &str,
) -> Option<String> {
collect_child_ids(parent).into_iter().find(|id| {
block_pool
.get(id)
.and_then(get_flavour)
.as_deref()
.is_some_and(|value| value == flavour)
})
}
packages/common/native/src/doc_parser/doc_loader.rs
-39
View File
@@ -1,39 +0,0 @@
use y_octo::{Doc, DocOptions};
use super::ParseError;
pub(super) fn is_empty_doc(binary: &[u8]) -> bool {
binary.is_empty() || binary == [0, 0]
}
pub(super) fn load_doc(binary: &[u8], doc_id: Option<&str>) -> Result<Doc, ParseError> {
if is_empty_doc(binary) {
return Err(ParseError::InvalidBinary);
}
let mut doc = build_doc(doc_id);
doc
.apply_update_from_binary_v1(binary)
.map_err(|_| ParseError::InvalidBinary)?;
Ok(doc)
}
pub(super) fn load_doc_or_new(binary: &[u8]) -> Result<Doc, ParseError> {
if is_empty_doc(binary) {
return Ok(DocOptions::new().build());
}
let mut doc = DocOptions::new().build();
doc
.apply_update_from_binary_v1(binary)
.map_err(|_| ParseError::InvalidBinary)?;
Ok(doc)
}
fn build_doc(doc_id: Option<&str>) -> Doc {
let options = DocOptions::new();
match doc_id {
Some(doc_id) => options.with_guid(doc_id.to_string()).build(),
None => options.build(),
}
}
packages/common/native/src/doc_parser/error.rs
-38
View File
@@ -1,38 +0,0 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;
use y_octo::JwstCodecError;
#[derive(Error, Debug, Serialize, Deserialize)]
pub enum ParseError {
#[error("doc_not_found")]
DocNotFound,
#[error("invalid_binary")]
InvalidBinary,
#[error("sqlite_error: {0}")]
SqliteError(String),
#[error("parser_error: {0}")]
ParserError(String),
#[error("unknown: {0}")]
Unknown(String),
}
impl From<JwstCodecError> for ParseError {
fn from(value: JwstCodecError) -> Self {
if matches!(
value,
JwstCodecError::DamagedDocumentJson
| JwstCodecError::IncompleteDocument(_)
| JwstCodecError::InvalidWriteBuffer(_)
| JwstCodecError::UpdateInvalid(_)
| JwstCodecError::StructClockInvalid { expect: _, actually: _ }
| JwstCodecError::StructSequenceInvalid { client_id: _, clock: _ }
| JwstCodecError::StructSequenceNotExists(_)
| JwstCodecError::RootStructNotFound(_)
| JwstCodecError::ParentNotFound
| JwstCodecError::IndexOutOfBound(_)
) {
return ParseError::InvalidBinary;
}
Self::ParserError(value.to_string())
}
}
packages/common/native/src/doc_parser/markdown/delta.rs
-842
View File
@@ -1,842 +0,0 @@
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
rc::{Rc, Weak},
};
use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value};
use super::{
super::value::{
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any,
},
inline::InlineStyle,
};
#[derive(Debug, Clone)]
struct InlineReference {
ref_type: Option<String>,
page_id: String,
title: Option<String>,
params: Option<AHashMap<String, Any>>,
mode: Option<String>,
}
#[derive(Debug, Clone)]
pub(crate) struct InlineReferencePayload {
pub(crate) doc_id: String,
pub(crate) payload: String,
}
#[derive(Debug, Clone)]
pub(crate) struct DeltaToMdOptions {
doc_url_prefix: Option<String>,
}
impl DeltaToMdOptions {
pub(crate) fn new(doc_url_prefix: Option<String>) -> Self {
Self { doc_url_prefix }
}
fn build_reference_link(&self, reference: &InlineReference) -> (String, String) {
let title = reference.title.clone().unwrap_or_default();
if let Some(prefix) = self.doc_url_prefix.as_deref() {
let prefix = prefix.trim_end_matches('/');
return (title, format!("{}/{}", prefix, reference.page_id));
}
let mut parts = Vec::new();
parts.push(reference.ref_type.clone().unwrap_or_else(|| "LinkedPage".into()));
parts.push(reference.page_id.clone());
if let Some(mode) = reference.mode.as_ref() {
parts.push(mode.clone());
}
(title, parts.join(":"))
}
}
pub(crate) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option<String> {
block
.get(key)
.and_then(|value| value.to_text())
.map(|text| delta_to_inline_markdown(&text, options))
}
pub(crate) fn delta_ops_to_markdown(ops: &[TextDeltaOp], options: &DeltaToMdOptions) -> String {
delta_to_markdown_with_options(ops, options, true)
}
pub(crate) fn delta_ops_to_plain_text(ops: &[TextDeltaOp]) -> String {
let mut out = String::new();
for op in ops {
if let TextDeltaOp::Insert {
insert: TextInsert::Text(text),
..
} = op
{
out.push_str(text);
}
}
out
}
pub(crate) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
let mut refs = Vec::new();
let mut seen: HashSet<(String, String)> = HashSet::new();
for op in delta {
let attrs = match op {
TextDeltaOp::Insert {
format: Some(format), ..
} => format,
_ => continue,
};
let reference = match attrs.get(InlineStyle::Reference.key()).and_then(parse_inline_reference) {
Some(reference) => reference,
None => continue,
};
let payload = match inline_reference_payload(&reference) {
Some(payload) => payload,
None => continue,
};
let key = (reference.page_id.clone(), payload.clone());
if seen.insert(key.clone()) {
refs.push(InlineReferencePayload {
doc_id: key.0,
payload: key.1,
});
}
}
refs
}
pub(crate) fn extract_inline_references_from_value(value: &Value) -> Vec<InlineReferencePayload> {
if let Some(text) = value.to_text() {
return extract_inline_references(&text.to_delta());
}
let Some(any) = value_to_any(value) else {
return Vec::new();
};
extract_inline_references_from_any(&any)
}
fn extract_inline_references_from_any(value: &Any) -> Vec<InlineReferencePayload> {
let Some(ops) = delta_ops_from_any(value) else {
return Vec::new();
};
let mut refs = Vec::new();
let mut seen: HashSet<(String, String)> = HashSet::new();
for op in ops {
let reference = match op
.attributes
.get(InlineStyle::Reference.key())
.and_then(parse_inline_reference)
{
Some(reference) => reference,
None => continue,
};
let payload = match inline_reference_payload(&reference) {
Some(payload) => payload,
None => continue,
};
let key = (reference.page_id.clone(), payload.clone());
if seen.insert(key.clone()) {
refs.push(InlineReferencePayload {
doc_id: key.0,
payload: key.1,
});
}
}
refs
}
fn parse_inline_reference(value: &Any) -> Option<InlineReference> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
let page_id = map.get("pageId").and_then(any_as_string).map(str::to_string)?;
let title = map.get("title").and_then(any_as_string).map(str::to_string);
let ref_type = map.get("type").and_then(any_as_string).map(str::to_string);
let params = map.get("params").and_then(|value| match value {
Any::Object(map) => Some(map.clone()),
_ => None,
});
let mode = params
.as_ref()
.and_then(|params| params.get("mode"))
.and_then(any_as_string)
.map(str::to_string);
Some(InlineReference {
ref_type,
page_id,
title,
params,
mode,
})
}
fn inline_reference_payload(reference: &InlineReference) -> Option<String> {
let params = reference.params.as_ref().map(params_any_map_to_json);
Some(build_reference_payload(&reference.page_id, params))
}
fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
delta_to_markdown_with_options(&text.to_delta(), options, false)
}
fn delta_to_markdown_with_options(delta: &[TextDeltaOp], options: &DeltaToMdOptions, trailing_newline: bool) -> String {
let ops = build_delta_ops(delta);
delta_ops_to_markdown_with_options(&ops, options, trailing_newline)
}
fn delta_ops_to_markdown_with_options(ops: &[DeltaOp], options: &DeltaToMdOptions, trailing_newline: bool) -> String {
let root = convert_delta_ops(ops, options);
let mut rendered = render_node(&root);
rendered = rendered.trim_end().to_string();
if trailing_newline {
rendered.push('\n');
}
rendered
}
#[derive(Debug, Clone)]
struct DeltaOp {
insert: DeltaInsert,
attributes: TextAttributes,
}
#[derive(Debug, Clone)]
enum DeltaInsert {
Text(String),
Embed(Vec<Any>),
}
fn delta_ops_from_any(value: &Any) -> Option<Vec<DeltaOp>> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
match map.get("$blocksuite:internal:text$") {
Some(Any::True) => {}
_ => return None,
}
let delta = map.get("delta")?;
let entries = match delta {
Any::Array(entries) => entries,
_ => return None,
};
let mut ops = Vec::new();
for entry in entries {
if let Some(op) = delta_op_from_any(entry) {
ops.push(op);
}
}
Some(ops)
}
fn delta_op_from_any(value: &Any) -> Option<DeltaOp> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
let insert_value = map.get("insert")?;
let insert = match insert_value {
Any::String(text) => DeltaInsert::Text(text.clone()),
Any::Array(values) => DeltaInsert::Embed(values.clone()),
_ => DeltaInsert::Embed(vec![insert_value.clone()]),
};
let attributes = map.get("attributes").and_then(any_to_attributes).unwrap_or_default();
Some(DeltaOp { insert, attributes })
}
fn any_to_attributes(value: &Any) -> Option<TextAttributes> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
let mut attrs = TextAttributes::new();
for (key, value) in map.iter() {
attrs.insert(key.clone(), value.clone());
}
Some(attrs)
}
fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Option<String> {
delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false))
}
pub(crate) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option<String> {
if let Some(text) = value.to_text() {
return Some(delta_to_inline_markdown(&text, options));
}
let any = value_to_any(value)?;
delta_any_to_inline_markdown(&any, options)
}
fn build_delta_ops(delta: &[TextDeltaOp]) -> Vec<DeltaOp> {
let mut ops = Vec::new();
for op in delta {
let (insert, attrs) = match op {
TextDeltaOp::Insert { insert, format } => (insert, format.clone().unwrap_or_default()),
_ => continue,
};
match insert {
TextInsert::Text(text) => ops.push(DeltaOp {
insert: DeltaInsert::Text(text.clone()),
attributes: attrs,
}),
TextInsert::Embed(values) => ops.push(DeltaOp {
insert: DeltaInsert::Embed(values.clone()),
attributes: attrs,
}),
}
}
ops
}
#[derive(Debug)]
struct Group {
node: Rc<RefCell<Node>>,
kind: String,
distance: usize,
count: usize,
}
fn convert_delta_ops(ops: &[DeltaOp], options: &DeltaToMdOptions) -> Rc<RefCell<Node>> {
let root = Node::new_root();
let mut group: Option<Group> = None;
let mut active_inline: HashMap<String, Any> = HashMap::new();
let mut beginning_of_line = false;
let mut line = Node::new_line();
let mut el = line.clone();
Node::append(&root, line.clone());
for index in 0..ops.len() {
let op = &ops[index];
let next_attrs = ops.get(index + 1).map(|next| &next.attributes);
match &op.insert {
DeltaInsert::Embed(values) => {
apply_inline_attributes(&mut el, &op.attributes, None, &mut active_inline, options);
for value in values {
match value {
Any::Object(map) => {
for (key, value) in map.iter() {
match key.as_str() {
"image" => {
if let Some(src) = any_as_string(value) {
let url = encode_link(src);
Node::append(&el, Node::new_text(&format!("![]({url})")));
}
}
"thematic_break" => {
let current_open = el.borrow().open.clone();
el.borrow_mut().open = format!("\n---\n{current_open}");
}
_ => {}
}
}
}
Any::String(value) => {
Node::append(&el, Node::new_text(value));
}
_ => {}
}
}
}
DeltaInsert::Text(text) => {
let lines: Vec<&str> = text.split('\n').collect();
if has_block_level_attribute(&op.attributes) {
for _ in 1..lines.len() {
for (attr, value) in op.attributes.iter() {
match attr.as_str() {
"header" => {
if let Some(level) = any_as_u64(value) {
let prefix = "#".repeat(level as usize);
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("{prefix} {current_open}");
new_line(&root, &mut line, &mut el, &mut active_inline);
break;
}
}
"blockquote" => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("> {current_open}");
new_line(&root, &mut line, &mut el, &mut active_inline);
break;
}
"list" => {
if group.as_ref().is_some_and(|g| g.kind != attr.as_str()) {
group = None;
}
if group.is_none() {
let group_node = Node::new_line();
Node::append(&root, group_node.clone());
group = Some(Group {
node: group_node,
kind: attr.to_string(),
distance: 0,
count: 0,
});
}
if let Some(group) = group.as_mut() {
Node::append(&group.node, line.clone());
group.distance = 0;
match any_as_string(value) {
Some("bullet") => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("- {current_open}");
}
Some("checked") => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("- [x] {current_open}");
}
Some("unchecked") => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("- [ ] {current_open}");
}
Some("ordered") => {
group.count += 1;
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("{}. {}", group.count, current_open);
}
_ => {}
}
}
new_line(&root, &mut line, &mut el, &mut active_inline);
break;
}
_ => {}
}
}
}
beginning_of_line = true;
} else {
for (line_index, segment) in lines.iter().enumerate() {
if (line_index > 0 || beginning_of_line) && group.is_some() {
let reset_group = group.as_mut().map(|group| {
group.distance += 1;
group.distance >= 2
});
if reset_group == Some(true) {
group = None;
}
}
apply_inline_attributes(&mut el, &op.attributes, next_attrs, &mut active_inline, options);
Node::append(&el, Node::new_text(segment));
if line_index + 1 < lines.len() {
new_line(&root, &mut line, &mut el, &mut active_inline);
}
}
beginning_of_line = false;
}
}
}
}
root
}
fn apply_inline_attributes(
el: &mut Rc<RefCell<Node>>,
attrs: &TextAttributes,
next: Option<&TextAttributes>,
active_inline: &mut HashMap<String, Any>,
options: &DeltaToMdOptions,
) {
let mut first = Vec::new();
let mut then = Vec::new();
let mut seen: HashSet<String> = HashSet::new();
let mut tag = el.clone();
loop {
let format = match tag.borrow().format.clone() {
Some(format) => format,
None => break,
};
seen.insert(format.clone());
let should_close = match attrs.get(&format) {
Some(value) => !any_truthy(value) || tag.borrow().open != tag.borrow().close,
None => true,
};
if should_close {
for key in seen.iter() {
active_inline.remove(key);
}
let parent = {
let tag_ref = tag.borrow();
tag_ref.parent.as_ref().and_then(|p| p.upgrade())
};
if let Some(parent) = parent {
*el = parent.clone();
tag = parent;
continue;
}
break;
}
let parent = {
let tag_ref = tag.borrow();
tag_ref.parent.as_ref().and_then(|p| p.upgrade())
};
if let Some(parent) = parent {
tag = parent;
} else {
break;
}
}
for (attr, value) in attrs.iter() {
if !is_inline_attribute(attr) || !any_truthy(value) {
continue;
}
if let Some(active) = active_inline.get(attr)
&& active == value
{
continue;
}
let next_matches = next
.and_then(|next_attrs| next_attrs.get(attr))
.map(|next_value| next_value == value)
.unwrap_or(false);
if next_matches {
first.push(attr.clone());
} else {
then.push(attr.clone());
}
active_inline.insert(attr.clone(), value.clone());
}
for attr in first.into_iter().chain(then) {
if let Some(node) = inline_node_for_attr(&attr, attrs, options) {
node.borrow_mut().format = Some(attr.clone());
Node::append(el, node.clone());
*el = node;
}
}
}
fn inline_node_for_attr(attr: &str, attrs: &TextAttributes, options: &DeltaToMdOptions) -> Option<Rc<RefCell<Node>>> {
let style = InlineStyle::from_key(attr)?;
if let Some(delimiter) = style.delimiter() {
return Some(Node::new_inline(delimiter.open, delimiter.close));
}
match style {
InlineStyle::Underline => Some(Node::new_inline("<u>", "</u>")),
InlineStyle::Color => {
let color = attrs.get(attr).and_then(any_as_string)?.trim();
if color.is_empty() {
None
} else {
Some(Node::new_inline(&format!("<span style=\"color: {color}\">"), "</span>"))
}
}
InlineStyle::Link => attrs
.get(attr)
.and_then(any_as_string)
.map(|url| Node::new_inline("[", &format!("]({url})"))),
InlineStyle::Reference => attrs.get(attr).and_then(parse_inline_reference).map(|reference| {
let (title, link) = options.build_reference_link(&reference);
Node::new_inline("[", &format!("{title}]({link})"))
}),
_ => None,
}
}
fn has_block_level_attribute(attrs: &TextAttributes) -> bool {
attrs.contains_key("header") || attrs.contains_key("blockquote") || attrs.contains_key("list")
}
fn is_inline_attribute(attr: &str) -> bool {
InlineStyle::from_key(attr).is_some()
}
fn encode_link(link: &str) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
#[inline]
fn push_pct(out: &mut String, b: u8) {
out.push('%');
out.push(HEX[(b >> 4) as usize] as char);
out.push(HEX[(b & 0x0f) as usize] as char);
}
#[inline]
fn is_allowed(b: u8) -> bool {
matches!(
b,
b'A'..=b'Z'
| b'a'..=b'z'
| b'0'..=b'9'
| b'-'
| b'_'
| b'.'
| b'!'
| b'~'
| b'*'
| b'\''
| b';'
| b','
| b'/'
| b'?'
| b':'
| b'@'
| b'&'
| b'='
| b'+'
| b'$'
| b'#'
)
}
let mut out = String::with_capacity(link.len());
for &b in link.as_bytes() {
match b {
b'(' | b')' => push_pct(&mut out, b),
b if is_allowed(b) => out.push(b as char),
b => push_pct(&mut out, b),
}
}
if let Some(i) = out.find("?response-content-disposition=attachment") {
out.truncate(i);
} else if let Some(i) = out.find("&response-content-disposition=attachment") {
out.truncate(i);
}
out
}
#[derive(Debug)]
struct Node {
open: String,
close: String,
text: String,
children: Vec<Rc<RefCell<Node>>>,
parent: Option<Weak<RefCell<Node>>>,
format: Option<String>,
}
impl Node {
fn new_root() -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: String::new(),
close: String::new(),
text: String::new(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn new_inline(open: &str, close: &str) -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: open.to_string(),
close: close.to_string(),
text: String::new(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn new_text(text: &str) -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: String::new(),
close: String::new(),
text: text.to_string(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn new_line() -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: String::new(),
close: "\n".to_string(),
text: String::new(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn append(parent: &Rc<RefCell<Node>>, child: Rc<RefCell<Node>>) {
if let Some(old_parent) = child.borrow().parent.as_ref().and_then(|p| p.upgrade()) {
let mut old_parent = old_parent.borrow_mut();
old_parent.children.retain(|existing| !Rc::ptr_eq(existing, &child));
}
child.borrow_mut().parent = Some(Rc::downgrade(parent));
parent.borrow_mut().children.push(child);
}
}
fn render_node(node: &Rc<RefCell<Node>>) -> String {
let node_ref = node.borrow();
let mut inner = node_ref.text.clone();
for child in node_ref.children.iter() {
inner.push_str(&render_node(child));
}
if inner.trim().is_empty()
&& node_ref.open == node_ref.close
&& !node_ref.open.is_empty()
&& !node_ref.close.is_empty()
{
return String::new();
}
let wrapped = !node_ref.open.is_empty() && !node_ref.close.is_empty();
let empty_inner = inner.trim().is_empty();
let mut fragments = Vec::new();
if inner.starts_with(' ') && !empty_inner && wrapped {
fragments.push(" ".to_string());
}
if !node_ref.open.is_empty() {
fragments.push(node_ref.open.clone());
}
fragments.push(if wrapped {
inner.trim().to_string()
} else {
inner.clone()
});
if !node_ref.close.is_empty() {
fragments.push(node_ref.close.clone());
}
if inner.ends_with(' ') && !empty_inner && wrapped {
fragments.push(" ".to_string());
}
fragments.join("")
}
fn new_line(
root: &Rc<RefCell<Node>>,
line: &mut Rc<RefCell<Node>>,
el: &mut Rc<RefCell<Node>>,
active_inline: &mut HashMap<String, Any>,
) {
*line = Node::new_line();
*el = line.clone();
Node::append(root, line.clone());
active_inline.clear();
}
#[cfg(test)]
mod tests {
use serde_json::Value;
use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert};
use super::*;
#[test]
fn test_delta_to_inline_markdown_link() {
let mut attrs = TextAttributes::new();
attrs.insert(
InlineStyle::Link.key().into(),
Any::String("https://example.com".into()),
);
let delta = vec![TextDeltaOp::Insert {
insert: TextInsert::Text("AFFiNE".into()),
format: Some(attrs),
}];
let options = DeltaToMdOptions::new(None);
let rendered = delta_to_markdown_with_options(&delta, &options, false);
assert_eq!(rendered, "[AFFiNE](https://example.com)");
}
#[test]
fn test_extract_inline_references_payload() {
let mut ref_map = AHashMap::default();
ref_map.insert("pageId".into(), Any::String("doc123".into()));
ref_map.insert("title".into(), Any::String("Doc Title".into()));
ref_map.insert("type".into(), Any::String("LinkedPage".into()));
let mut attrs = TextAttributes::new();
attrs.insert(InlineStyle::Reference.key().into(), Any::Object(ref_map));
let delta = vec![TextDeltaOp::Insert {
insert: TextInsert::Text("Doc Title".into()),
format: Some(attrs),
}];
let refs = extract_inline_references(&delta);
assert_eq!(refs.len(), 1);
assert_eq!(refs[0].doc_id, "doc123");
let payload: Value = serde_json::from_str(&refs[0].payload).unwrap();
assert_eq!(payload, serde_json::json!({ "docId": "doc123" }));
}
#[test]
fn test_delta_to_inline_markdown_underline() {
let mut attrs = TextAttributes::new();
attrs.insert(InlineStyle::Underline.key().into(), Any::True);
let delta = vec![TextDeltaOp::Insert {
insert: TextInsert::Text("Under".into()),
format: Some(attrs),
}];
let options = DeltaToMdOptions::new(None);
let rendered = delta_to_markdown_with_options(&delta, &options, false);
assert_eq!(rendered, "<u>Under</u>");
}
#[test]
fn test_delta_to_inline_markdown_color() {
let mut attrs = TextAttributes::new();
attrs.insert(InlineStyle::Color.key().into(), Any::String("red".into()));
let delta = vec![TextDeltaOp::Insert {
insert: TextInsert::Text("Red".into()),
format: Some(attrs),
}];
let options = DeltaToMdOptions::new(None);
let rendered = delta_to_markdown_with_options(&delta, &options, false);
assert_eq!(rendered, "<span style=\"color: red\">Red</span>");
}
}
packages/common/native/src/doc_parser/markdown/inline.rs
-71
View File
@@ -1,71 +0,0 @@
const INLINE_ATTR_BOLD: &str = "bold";
const INLINE_ATTR_ITALIC: &str = "italic";
const INLINE_ATTR_UNDERLINE: &str = "underline";
const INLINE_ATTR_STRIKE: &str = "strike";
const INLINE_ATTR_CODE: &str = "code";
const INLINE_ATTR_LINK: &str = "link";
const INLINE_ATTR_REFERENCE: &str = "reference";
const INLINE_ATTR_COLOR: &str = "color";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum InlineStyle {
Bold,
Italic,
Underline,
Strike,
Code,
Link,
Reference,
Color,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct InlineDelimiter {
pub(super) open: &'static str,
pub(super) close: &'static str,
}
impl InlineStyle {
pub(super) fn key(self) -> &'static str {
match self {
InlineStyle::Bold => INLINE_ATTR_BOLD,
InlineStyle::Italic => INLINE_ATTR_ITALIC,
InlineStyle::Underline => INLINE_ATTR_UNDERLINE,
InlineStyle::Strike => INLINE_ATTR_STRIKE,
InlineStyle::Code => INLINE_ATTR_CODE,
InlineStyle::Link => INLINE_ATTR_LINK,
InlineStyle::Reference => INLINE_ATTR_REFERENCE,
InlineStyle::Color => INLINE_ATTR_COLOR,
}
}
pub(super) fn from_key(key: &str) -> Option<Self> {
match key {
INLINE_ATTR_BOLD => Some(InlineStyle::Bold),
INLINE_ATTR_ITALIC => Some(InlineStyle::Italic),
INLINE_ATTR_UNDERLINE => Some(InlineStyle::Underline),
INLINE_ATTR_STRIKE => Some(InlineStyle::Strike),
INLINE_ATTR_CODE => Some(InlineStyle::Code),
INLINE_ATTR_LINK => Some(InlineStyle::Link),
INLINE_ATTR_REFERENCE => Some(InlineStyle::Reference),
INLINE_ATTR_COLOR => Some(InlineStyle::Color),
_ => None,
}
}
pub(super) fn delimiter(self) -> Option<InlineDelimiter> {
match self {
InlineStyle::Italic => Some(InlineDelimiter { open: "_", close: "_" }),
InlineStyle::Bold => Some(InlineDelimiter {
open: "**",
close: "**",
}),
InlineStyle::Strike => Some(InlineDelimiter {
open: "~~",
close: "~~",
}),
InlineStyle::Code => Some(InlineDelimiter { open: "`", close: "`" }),
InlineStyle::Link | InlineStyle::Reference | InlineStyle::Underline | InlineStyle::Color => None,
}
}
}
packages/common/native/src/doc_parser/markdown/mod.rs
-13
View File
@@ -1,13 +0,0 @@
mod delta;
mod inline;
mod parser;
mod render;
pub(crate) use delta::{
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references,
extract_inline_references_from_value, text_to_inline_markdown,
};
#[cfg(test)]
pub(crate) use parser::MAX_MARKDOWN_CHARS;
pub(crate) use parser::{MAX_BLOCKS, parse_markdown_blocks};
pub(crate) use render::{MarkdownRenderer, MarkdownWriter};
packages/common/native/src/doc_parser/markdown/parser.rs
-1723
View File
File diff suppressed because it is too large Load Diff
packages/common/native/src/doc_parser/markdown/render.rs
-235
View File
@@ -1,235 +0,0 @@
use super::{
super::block_spec::{BlockFlavour, BlockSpec},
delta::{DeltaToMdOptions, delta_ops_to_markdown, delta_ops_to_plain_text},
};
pub(crate) struct MarkdownWriter<'a> {
output: &'a mut String,
}
impl<'a> MarkdownWriter<'a> {
pub(crate) fn new(output: &'a mut String) -> Self {
Self { output }
}
pub(crate) fn push_paragraph(&mut self, prefix: &str, text: &str) {
if prefix == "> " {
let quoted = text
.split('\n')
.map(|line| format!("{prefix}{line}"))
.collect::<Vec<_>>()
.join("\n");
self.output.push_str(&quoted);
if !text.ends_with('\n') {
self.output.push('\n');
}
self.output.push('\n');
return;
}
self.output.push_str(prefix);
self.output.push_str(text);
if !text.ends_with('\n') {
self.output.push('\n');
}
self.output.push('\n');
}
pub(crate) fn push_list_item(&mut self, indent: &str, prefix: &str, text: &str) {
self.output.push_str(indent);
self.output.push_str(prefix);
self.output.push_str(text);
if !text.ends_with('\n') {
self.output.push('\n');
}
}
pub(crate) fn push_code_block(&mut self, lang: &str, text: &str) {
self.output.push_str("```");
self.output.push_str(lang);
self.output.push('\n');
self.output.push_str(text);
self.output.push_str("\n```\n\n");
}
pub(crate) fn push_divider(&mut self) {
self.output.push_str("\n---\n\n");
}
pub(crate) fn push_table(&mut self, table: &str) {
if table.is_empty() {
self.output.push('\n');
return;
}
self.output.push_str(table);
if !table.ends_with('\n') {
self.output.push('\n');
}
self.output.push('\n');
}
}
pub(crate) fn paragraph_prefix(type_: &str) -> &'static str {
match type_ {
"h1" => "# ",
"h2" => "## ",
"h3" => "### ",
"h4" => "#### ",
"h5" => "##### ",
"h6" => "###### ",
"quote" => "> ",
_ => "",
}
}
pub(crate) fn list_prefix(r#type: &str, checked: bool, order: Option<i64>) -> String {
match r#type {
"bulleted" => "* ".to_string(),
"todo" => if checked { "- [x] " } else { "- [ ] " }.to_string(),
_ => format!("{}. ", order.unwrap_or(1)),
}
}
pub(crate) fn list_indent(depth: usize) -> String {
" ".repeat(depth)
}
pub(crate) struct MarkdownRenderer<'a> {
options: &'a DeltaToMdOptions,
}
impl<'a> MarkdownRenderer<'a> {
pub(crate) fn new(options: &'a DeltaToMdOptions) -> Self {
Self { options }
}
pub(crate) fn write_block(&self, output: &mut String, spec: &BlockSpec, list_depth: usize) {
match spec.flavour {
BlockFlavour::Paragraph => {
let prefix = paragraph_prefix(spec.block_type_str().unwrap_or_default());
let text_md = delta_ops_to_markdown(&spec.text, self.options);
let mut writer = MarkdownWriter::new(output);
writer.push_paragraph(prefix, &text_md);
}
BlockFlavour::List => {
let type_ = spec.block_type_str().unwrap_or("bulleted");
let checked = spec.checked.unwrap_or(false);
let prefix = list_prefix(type_, checked, spec.order);
let indent = list_indent(list_depth);
let text_md = delta_ops_to_markdown(&spec.text, self.options);
let mut writer = MarkdownWriter::new(output);
writer.push_list_item(&indent, &prefix, &text_md);
}
BlockFlavour::Code => {
let text = delta_ops_to_plain_text(&spec.text);
let lang = spec.language.as_deref().unwrap_or_default();
let mut writer = MarkdownWriter::new(output);
writer.push_code_block(lang, &text);
}
BlockFlavour::Divider => {
let mut writer = MarkdownWriter::new(output);
writer.push_divider();
}
BlockFlavour::Image => {
if let Some(image) = spec.image.as_ref() {
output.push_str(&image.render_markdown());
}
}
BlockFlavour::Bookmark => {
if let Some(bookmark) = spec.bookmark.as_ref() {
output.push_str(&format!("\n[](Bookmark,{})\n\n", bookmark.url));
}
}
BlockFlavour::EmbedYoutube => {
if let Some(embed) = spec.embed_youtube.as_ref() {
output.push_str(
&format!(
"\n <iframe\n type=\"text/html\"\n width=\"100%\"\n height=\"410px\"\n src=\"https://www.youtube.com/embed/{}\"\n frameborder=\"0\"\n allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\"\n allowfullscreen\n credentialless>\n </iframe>\n\n",
embed.video_id
)
);
}
}
BlockFlavour::EmbedIframe => {
if let Some(embed) = spec.embed_iframe.as_ref() {
output.push_str(&format!("\n<iframe src=\"{}\"></iframe>\n\n", embed.url));
}
}
BlockFlavour::Callout => {}
BlockFlavour::Table => {
if let Some(table) = spec.table.as_ref()
&& let Some(table_md) = table.render_markdown()
{
let mut writer = MarkdownWriter::new(output);
writer.push_table(&table_md);
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_paragraph_newlines() {
let mut markdown = String::new();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_paragraph("# ", "Title\n");
}
assert_eq!(markdown, "# Title\n\n");
markdown.clear();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_paragraph("", "Plain");
}
assert_eq!(markdown, "Plain\n\n");
}
#[test]
fn test_list_newlines() {
let mut markdown = String::new();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_list_item(" ", "* ", "Item\n");
}
assert_eq!(markdown, " * Item\n");
markdown.clear();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_list_item("", "- [ ] ", "Task");
}
assert_eq!(markdown, "- [ ] Task\n");
}
#[test]
fn test_code_block_newlines() {
let mut markdown = String::new();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_code_block("rs", "fn main() {}");
}
assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n");
}
#[test]
fn test_table_newlines() {
let mut markdown = String::new();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_table("|a|b|\n|---|---|\n|1|2|\n");
}
assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n");
markdown.clear();
{
let mut writer = MarkdownWriter::new(&mut markdown);
writer.push_table("|a|b|");
}
assert_eq!(markdown, "|a|b|\n\n");
}
}
packages/common/native/src/doc_parser/mod.rs
-22
View File
@@ -1,22 +0,0 @@
mod block_spec;
mod blocksuite;
mod doc_loader;
mod error;
mod markdown;
mod read;
#[cfg(test)]
mod roundtrip_tests;
mod schema;
mod table;
mod value;
mod write;
pub use error::ParseError;
pub use read::{
BlockInfo, CrawlResult, MarkdownResult, PageDocContent, WorkspaceDocContent, get_doc_ids_from_binary,
parse_doc_from_binary, parse_doc_to_markdown, parse_page_doc, parse_workspace_doc,
};
pub use write::{
add_doc_to_root_doc, build_full_doc, build_public_root_doc, update_doc, update_doc_properties, update_doc_title,
update_root_doc_meta_title,
};
packages/common/native/src/doc_parser/read/database.rs
-286
View File
@@ -1,286 +0,0 @@
use std::collections::HashSet;
use y_octo::{Any, Map, Value};
use super::{text_content, text_content_for_summary};
use crate::doc_parser::{
blocksuite::{DocContext, collect_child_ids, get_string},
markdown::{
DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references_from_value,
text_to_inline_markdown,
},
table::{MarkdownTableOptions, render_markdown_table},
value::{any_as_string, value_to_string},
};
pub(super) struct DatabaseTable {
pub(super) columns: Vec<DatabaseColumn>,
pub(super) rows: Vec<Vec<String>>,
}
pub(super) struct DatabaseOption {
id: Option<String>,
value: Option<String>,
color: Option<String>,
}
pub(super) struct DatabaseColumn {
pub(super) id: String,
pub(super) name: Option<String>,
pub(super) col_type: String,
pub(super) options: Vec<DatabaseOption>,
}
pub(super) fn build_database_table(
block: &Map,
context: &DocContext,
md_options: &DeltaToMdOptions,
) -> Option<DatabaseTable> {
let columns = parse_database_columns(block)?;
let cells_map = block.get("prop:cells").and_then(|v| v.to_map())?;
let child_ids = collect_child_ids(block);
let mut rows = Vec::new();
for child_id in child_ids {
let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map());
let mut row = Vec::new();
for column in columns.iter() {
let mut cell_text = String::new();
if column.col_type == "title" {
if let Some(child_block) = context.block_pool.get(&child_id) {
if let Some(text_md) = text_to_inline_markdown(child_block, "prop:text", md_options) {
cell_text = text_md;
} else if let Some((text, _)) = text_content(child_block, "prop:text") {
cell_text = text;
} else if let Some((text, _)) = text_content_for_summary(child_block, "prop:text") {
cell_text = text;
}
}
} else if let Some(row_cells) = &row_cells
&& let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map())
&& let Some(value) = cell_val.get("value")
{
if let Some(text_md) = delta_value_to_inline_markdown(&value, md_options) {
cell_text = text_md;
} else {
cell_text = format_cell_value(&value, column);
}
}
row.push(cell_text);
}
if row.iter().any(|cell| !cell.is_empty()) {
rows.push(row);
}
}
Some(DatabaseTable { columns, rows })
}
pub(super) fn database_table_markdown(table: DatabaseTable) -> Option<String> {
let mut rows = Vec::with_capacity(table.rows.len() + 1);
let header = table
.columns
.iter()
.map(|column| column.name.as_deref().unwrap_or_default().to_string())
.collect::<Vec<_>>();
rows.push(header);
rows.extend(table.rows);
let options = MarkdownTableOptions::new(false, "<br />", true);
render_markdown_table(&rows, options)
}
pub(super) fn database_summary_text(block: &Map, context: &DocContext) -> Option<String> {
let md_options = DeltaToMdOptions::new(None);
let table = build_database_table(block, context, &md_options)?;
let mut summary = String::new();
if let Some(title) = get_string(block, "prop:title")
&& !title.is_empty()
{
summary.push_str(&title);
summary.push('|');
}
for column in table.columns.iter() {
if let Some(name) = column.name.as_ref()
&& !name.is_empty()
{
summary.push_str(name);
summary.push('|');
}
for option in column.options.iter() {
if let Some(value) = option.value.as_ref()
&& !value.is_empty()
{
summary.push_str(value);
summary.push('|');
}
}
}
for row in table.rows.iter() {
for cell_text in row.iter() {
if !cell_text.is_empty() {
summary.push_str(cell_text);
summary.push('|');
}
}
}
if summary.is_empty() { None } else { Some(summary) }
}
pub(super) fn gather_database_texts(block: &Map) -> (Vec<String>, Option<String>) {
let mut texts = Vec::new();
let database_title = get_string(block, "prop:title");
if let Some(title) = &database_title {
texts.push(title.clone());
}
if let Some(columns) = parse_database_columns(block) {
for column in columns.iter() {
if let Some(name) = column.name.as_ref() {
texts.push(name.clone());
}
for option in column.options.iter() {
if let Some(value) = option.value.as_ref() {
texts.push(value.clone());
}
}
}
}
(texts, database_title)
}
pub(super) fn collect_database_cell_references(block: &Map) -> Vec<InlineReferencePayload> {
let cells_map = match block.get("prop:cells").and_then(|value| value.to_map()) {
Some(map) => map,
None => return Vec::new(),
};
let mut refs = Vec::new();
let mut seen: HashSet<(String, String)> = HashSet::new();
for row in cells_map.values() {
let Some(row_map) = row.to_map() else {
continue;
};
for cell in row_map.values() {
let Some(cell_map) = cell.to_map() else {
continue;
};
let Some(value) = cell_map.get("value") else {
continue;
};
for reference in extract_inline_references_from_value(&value) {
let key = (reference.doc_id.clone(), reference.payload.clone());
if seen.insert(key) {
refs.push(reference);
}
}
}
}
refs
}
fn parse_database_columns(block: &Map) -> Option<Vec<DatabaseColumn>> {
let columns = block.get("prop:columns").and_then(|value| value.to_array())?;
let mut parsed = Vec::new();
for column_value in columns.iter() {
if let Some(column) = column_value.to_map() {
let id = column
.get("id")
.and_then(|value| value_to_string(&value))
.unwrap_or_default();
let name = column.get("name").and_then(|value| value_to_string(&value));
let col_type = column
.get("type")
.and_then(|value| value_to_string(&value))
.unwrap_or_default();
let options = parse_database_options(&column);
parsed.push(DatabaseColumn {
id,
name,
col_type,
options,
});
}
}
Some(parsed)
}
fn parse_database_options(column: &Map) -> Vec<DatabaseOption> {
let Some(data) = column.get("data").and_then(|value| value.to_map()) else {
return Vec::new();
};
let Some(options) = data.get("options").and_then(|value| value.to_array()) else {
return Vec::new();
};
let mut parsed = Vec::new();
for option_value in options.iter() {
if let Some(option) = option_value.to_map() {
parsed.push(DatabaseOption {
id: option.get("id").and_then(|value| value_to_string(&value)),
value: option.get("value").and_then(|value| value_to_string(&value)),
color: option.get("color").and_then(|value| value_to_string(&value)),
});
}
}
parsed
}
fn format_option_tag(option: &DatabaseOption) -> String {
let id = option.id.as_deref().unwrap_or_default();
let value = option.value.as_deref().unwrap_or_default();
let color = option.color.as_deref().unwrap_or_default();
format!("<span data-affine-option data-value=\"{id}\" data-option-color=\"{color}\">{value}</span>")
}
fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String {
match column.col_type.as_str() {
"select" => {
let id = match value {
Value::Any(any) => any_as_string(any).map(str::to_string),
Value::Text(text) => Some(text.to_string()),
_ => None,
};
if let Some(id) = id {
for option in column.options.iter() {
if option.id.as_deref() == Some(id.as_str()) {
return format_option_tag(option);
}
}
}
String::new()
}
"multi-select" => {
let ids: Vec<String> = match value {
Value::Any(Any::Array(ids)) => ids.iter().filter_map(any_as_string).map(str::to_string).collect(),
Value::Array(array) => array.iter().filter_map(|id_val| value_to_string(&id_val)).collect(),
_ => Vec::new(),
};
if ids.is_empty() {
return String::new();
}
let mut selected = Vec::new();
for id in ids.iter() {
for option in column.options.iter() {
if option.id.as_deref() == Some(id.as_str()) {
selected.push(format_option_tag(option));
}
}
}
selected.join("")
}
_ => value_to_string(value).unwrap_or_default(),
}
}
packages/common/native/src/doc_parser/read/mod.rs
-985
View File
@@ -1,985 +0,0 @@
mod database;
use std::collections::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use serde_json::{Map as JsonMap, Value as JsonValue};
use y_octo::{Any, Map};
use self::database::{
build_database_table, collect_database_cell_references, database_summary_text, database_table_markdown,
gather_database_texts,
};
use super::{
ParseError,
block_spec::{BlockFlavour, BlockSpec, ImageSpec},
blocksuite::{DocContext, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour},
doc_loader::load_doc,
markdown::{DeltaToMdOptions, MarkdownRenderer, MarkdownWriter, extract_inline_references},
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR},
value::{any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string},
};
const SUMMARY_LIMIT: usize = 1000;
const DEFAULT_PAGE_TITLE: &str = "Untitled";
const KNOWN_UNSUPPORTED_MARKDOWN_FLAVOURS: [&str; 10] = [
"affine:attachment",
"affine:callout",
"affine:note",
"affine:edgeless-text",
"affine:embed-linked-doc",
"affine:embed-synced-doc",
"affine:frame",
"affine:latex",
"affine:surface",
"affine:surface-ref",
];
const BOOKMARK_FLAVOURS: [&str; 6] = [
"affine:bookmark",
"affine:embed-youtube",
"affine:embed-iframe",
"affine:embed-figma",
"affine:embed-github",
"affine:embed-loom",
];
struct SummaryBuilder {
summary: String,
remaining: Option<isize>,
}
impl SummaryBuilder {
fn new(limit: isize) -> Self {
let remaining = if limit < 0 { None } else { Some(limit) };
Self {
summary: String::new(),
remaining,
}
}
fn is_unlimited(&self) -> bool {
self.remaining.is_none()
}
fn push_text(&mut self, text: &str, len: usize) {
match self.remaining {
None => self.summary.push_str(text),
Some(remaining) if remaining > 0 => {
self.summary.push_str(text);
self.remaining = Some(remaining - len as isize);
}
_ => {}
}
}
fn push_raw(&mut self, text: &str) {
self.summary.push_str(text);
}
fn into_string(self) -> String {
self.summary
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BlockInfo {
pub block_id: String,
pub flavour: String,
pub content: Option<Vec<String>>,
pub blob: Option<Vec<String>>,
pub ref_doc_id: Option<Vec<String>>,
pub ref_info: Option<Vec<String>>,
pub parent_flavour: Option<String>,
pub parent_block_id: Option<String>,
pub additional: Option<String>,
}
impl BlockInfo {
fn base(
block_id: &str,
flavour: &str,
parent_flavour: Option<&String>,
parent_block_id: Option<&String>,
additional: Option<String>,
) -> Self {
Self {
block_id: block_id.to_string(),
flavour: flavour.to_string(),
content: None,
blob: None,
ref_doc_id: None,
ref_info: None,
parent_flavour: parent_flavour.cloned(),
parent_block_id: parent_block_id.cloned(),
additional,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CrawlResult {
pub blocks: Vec<BlockInfo>,
pub title: String,
pub summary: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageDocContent {
pub title: String,
pub summary: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkspaceDocContent {
pub name: String,
#[serde(rename = "avatarKey")]
pub avatar_key: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MarkdownResult {
pub title: String,
pub markdown: String,
pub known_unsupported_blocks: Vec<String>,
pub unknown_blocks: Vec<String>,
}
fn is_known_unsupported_markdown_flavour(flavour: &str) -> bool {
KNOWN_UNSUPPORTED_MARKDOWN_FLAVOURS.contains(&flavour) || flavour.starts_with("affine:edgeless-")
}
fn is_edgeless_markdown_flavour(flavour: &str) -> bool {
matches!(flavour, "affine:surface" | "affine:frame" | "affine:surface-ref") || flavour.starts_with("affine:edgeless-")
}
fn has_skipped_markdown_ancestor(
block_id: &str,
parent_lookup: &HashMap<String, String>,
skipped_subtrees: &HashSet<String>,
) -> bool {
let mut cursor = parent_lookup.get(block_id).cloned();
while let Some(parent_id) = cursor {
if skipped_subtrees.contains(&parent_id) {
return true;
}
cursor = parent_lookup.get(&parent_id).cloned();
}
false
}
pub fn parse_workspace_doc(doc_bin: Vec<u8>) -> Result<Option<WorkspaceDocContent>, ParseError> {
let doc = load_doc(&doc_bin, None)?;
let meta = match doc.get_map("meta") {
Ok(meta) => meta,
Err(_) => return Ok(None),
};
let name = get_string(&meta, "name").unwrap_or_default();
let avatar_key = get_string(&meta, "avatar").unwrap_or_default();
Ok(Some(WorkspaceDocContent { name, avatar_key }))
}
pub fn parse_page_doc(
doc_bin: Vec<u8>,
max_summary_length: Option<isize>,
) -> Result<Option<PageDocContent>, ParseError> {
let doc = load_doc(&doc_bin, None)?;
let blocks_map = match doc.get_map("blocks") {
Ok(map) => map,
Err(_) => return Ok(None),
};
if blocks_map.is_empty() {
return Ok(None);
}
let Some(context) = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) else {
return Ok(None);
};
let mut walker = context.walker();
let mut content = PageDocContent {
title: context
.block_pool
.get(&context.root_block_id)
.and_then(|block| get_string(block, "prop:title"))
.unwrap_or_default(),
summary: String::new(),
};
let mut summary = SummaryBuilder::new(max_summary_length.unwrap_or(150));
while let Some((_parent_block_id, block_id)) = walker.next() {
let Some(block) = context.block_pool.get(&block_id) else {
continue;
};
let Some(flavour) = get_flavour(block) else {
continue;
};
match flavour.as_str() {
"affine:page" | "affine:note" => {
walker.enqueue_children(&block_id, block);
}
"affine:attachment" | "affine:transcription" | "affine:callout" => {
if summary.is_unlimited() {
walker.enqueue_children(&block_id, block);
}
}
"affine:database" => {
if summary.is_unlimited()
&& let Some(text) = database_summary_text(block, &context)
{
summary.push_raw(&text);
}
}
"affine:table" => {
if summary.is_unlimited() {
let contents = table_cell_texts(block);
if !contents.is_empty() {
summary.push_raw(&contents.join("|"));
}
}
}
"affine:paragraph" | "affine:list" | "affine:code" => {
walker.enqueue_children(&block_id, block);
if let Some((text, len)) = text_content_for_summary(block, "prop:text") {
summary.push_text(&text, len);
}
}
_ => {}
}
}
content.summary = summary.into_string();
Ok(Some(content))
}
pub fn parse_doc_to_markdown(
doc_bin: Vec<u8>,
doc_id: String,
ai_editable: bool,
doc_url_prefix: Option<String>,
) -> Result<MarkdownResult, ParseError> {
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
let blocks_map = doc.get_map("blocks")?;
if blocks_map.is_empty() {
return Ok(MarkdownResult {
title: "".into(),
markdown: "".into(),
known_unsupported_blocks: vec![],
unknown_blocks: vec![],
});
}
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
let root_block_id = context.root_block_id.clone();
let mut walker = context.walker();
let mut doc_title = String::from(DEFAULT_PAGE_TITLE);
let mut markdown = String::new();
let mut known_unsupported_blocks = Vec::new();
let mut unknown_blocks = Vec::new();
let mut skipped_subtrees = HashSet::new();
let md_options = DeltaToMdOptions::new(doc_url_prefix);
let renderer = MarkdownRenderer::new(&md_options);
while let Some((_parent_block_id, block_id)) = walker.next() {
let block = match context.block_pool.get(&block_id) {
Some(block) => block,
None => continue,
};
let flavour = match get_flavour(block) {
Some(flavour) => flavour,
None => continue,
};
if flavour == PAGE_FLAVOUR {
// enqueue children first to keep traversal order similar to JS implementation
walker.enqueue_children(&block_id, block);
let title = get_string(block, "prop:title").unwrap_or_default();
doc_title = title.clone();
continue;
}
let parent_id = context.parent_lookup.get(&block_id);
let parent_flavour = parent_id
.and_then(|id| context.block_pool.get(id))
.and_then(get_flavour);
if parent_flavour.as_deref() == Some("affine:database") {
continue;
}
// enqueue children first to keep traversal order similar to JS implementation
walker.enqueue_children(&block_id, block);
if is_known_unsupported_markdown_flavour(flavour.as_str()) {
known_unsupported_blocks.push(format!("{block_id}:{flavour}"));
if is_edgeless_markdown_flavour(flavour.as_str()) {
skipped_subtrees.insert(block_id.clone());
}
continue;
}
if BlockFlavour::from_str(flavour.as_str()).is_none() && flavour.as_str() != "affine:database" {
unknown_blocks.push(format!("{block_id}:{flavour}"));
skipped_subtrees.insert(block_id.clone());
continue;
}
if has_skipped_markdown_ancestor(&block_id, &context.parent_lookup, &skipped_subtrees) {
continue;
}
let block_level = if ai_editable {
block_level(&block_id, &root_block_id, &context.parent_lookup)
} else {
0
};
let ai_block = ai_editable && block_level == 2;
let mut block_markdown = String::new();
match flavour.as_str() {
"affine:database" => {
let title = get_string(block, "prop:title").unwrap_or_default();
block_markdown.push_str(&format!("\n### {title}\n"));
if let Some(table) = build_database_table(block, &context, &md_options)
&& let Some(table_md) = database_table_markdown(table)
{
let mut writer = MarkdownWriter::new(&mut block_markdown);
writer.push_table(&table_md);
}
}
_ => {
let Some(block_flavour) = BlockFlavour::from_str(flavour.as_str()) else {
continue;
};
let spec = BlockSpec::from_block_map_with_flavour(block, block_flavour);
let list_depth = if block_flavour == BlockFlavour::List {
get_list_depth(&block_id, &context.parent_lookup, &context.block_pool)
} else {
0
};
renderer.write_block(&mut block_markdown, &spec, list_depth);
}
}
if ai_block {
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} -->\n"));
}
markdown.push_str(&block_markdown);
}
Ok(MarkdownResult {
title: doc_title,
markdown,
known_unsupported_blocks,
unknown_blocks,
})
}
pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlResult, ParseError> {
let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?;
let blocks_map = doc.get_map("blocks")?;
if blocks_map.is_empty() {
return Err(ParseError::ParserError("blocks map is empty".into()));
}
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
let mut walker = context.walker();
let mut blocks: Vec<BlockInfo> = Vec::with_capacity(context.block_pool.len());
let mut doc_title = String::new();
let mut summary = SummaryBuilder::new(SUMMARY_LIMIT as isize);
while let Some((parent_block_id, block_id)) = walker.next() {
let block = match context.block_pool.get(&block_id) {
Some(block) => block,
None => continue,
};
let flavour = match get_flavour(block) {
Some(flavour) => flavour,
None => continue,
};
let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id));
let parent_flavour = parent_block.and_then(get_flavour);
let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool);
let note_block_id = note_block.as_ref().and_then(get_block_id);
let display_mode = determine_display_mode(note_block.as_ref());
// enqueue children first to keep traversal order similar to JS implementation
walker.enqueue_children(&block_id, block);
let build_block = |database_name: Option<&String>| {
BlockInfo::base(
&block_id,
&flavour,
parent_flavour.as_ref(),
parent_block_id.as_ref(),
compose_additional(&display_mode, note_block_id.as_ref(), database_name),
)
};
if flavour == PAGE_FLAVOUR {
let title = get_string(block, "prop:title").unwrap_or_default();
doc_title = title.clone();
let mut info = build_block(None);
info.content = Some(vec![title]);
blocks.push(info);
continue;
}
if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") {
if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) {
let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") {
parent_block.and_then(|map| get_string(map, "prop:title"))
} else {
None
};
let content = text.to_string();
let text_len = text.len() as usize;
let refs = extract_inline_references(&text.to_delta());
let mut info = build_block(database_name.as_ref());
info.content = Some(vec![content.clone()]);
if !refs.is_empty() {
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
}
blocks.push(info);
summary.push_text(&content, text_len);
}
continue;
}
if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") {
if let Some(page_id) = get_string(block, "prop:pageId") {
let mut info = build_block(None);
let payload = embed_ref_payload(block, &page_id);
apply_doc_ref(&mut info, page_id, payload);
blocks.push(info);
}
continue;
}
if flavour == "affine:attachment" {
if let Some(blob_id) = get_string(block, "prop:sourceId") {
let mut info = build_block(None);
let name = get_string(block, "prop:name").unwrap_or_default();
apply_blob_info(&mut info, blob_id, name);
blocks.push(info);
}
continue;
}
if flavour == "affine:image" {
let image = ImageSpec::from_block_map(block);
if !image.source_id.is_empty() {
let mut info = build_block(None);
let caption = image.caption.unwrap_or_default();
apply_blob_info(&mut info, image.source_id, caption);
blocks.push(info);
}
continue;
}
if flavour == "affine:surface" {
let texts = gather_surface_texts(block);
let mut info = build_block(None);
info.content = Some(texts);
blocks.push(info);
continue;
}
if flavour == "affine:database" {
let (texts, database_name) = gather_database_texts(block);
let mut info = BlockInfo::base(
&block_id,
&flavour,
parent_flavour.as_ref(),
parent_block_id.as_ref(),
compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()),
);
info.content = Some(texts);
let refs = collect_database_cell_references(block);
if !refs.is_empty() {
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
}
blocks.push(info);
continue;
}
if flavour == "affine:latex" {
if let Some(content) = get_string(block, "prop:latex") {
let mut info = build_block(None);
info.content = Some(vec![content]);
blocks.push(info);
}
continue;
}
if flavour == "affine:table" {
let contents = table_cell_texts(block);
let mut info = build_block(None);
info.content = Some(contents);
blocks.push(info);
continue;
}
if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) {
blocks.push(build_block(None));
}
}
if doc_title.is_empty() {
doc_title = DEFAULT_PAGE_TITLE.into();
}
Ok(CrawlResult {
blocks,
title: doc_title,
summary: summary.into_string(),
})
}
pub fn get_doc_ids_from_binary(doc_bin: Vec<u8>, include_trash: bool) -> Result<Vec<String>, ParseError> {
let doc = load_doc(&doc_bin, None)?;
let mut doc_ids = Vec::new();
let meta = doc.get_map("meta")?;
let pages_value = meta.get("pages");
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
for page_val in pages.iter() {
if let Some(page) = page_val.to_map() {
let id = get_string(&page, "id");
if let Some(id) = id {
let trash = page
.get("trash")
.and_then(|v| match v.to_any() {
Some(Any::True) => Some(true),
Some(Any::False) => Some(false),
_ => None,
})
.unwrap_or(false);
if include_trash || !trash {
doc_ids.push(id);
}
}
}
}
return Ok(doc_ids);
}
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
for entry in entries {
let Any::Object(map) = entry else {
continue;
};
let id = map.get("id").and_then(any_as_string).map(str::to_string);
if let Some(id) = id {
let trash = map.get("trash").map(any_truthy).unwrap_or(false);
if include_trash || !trash {
doc_ids.push(id);
}
}
}
}
Ok(doc_ids)
}
fn block_level(block_id: &str, root_id: &str, parent_lookup: &HashMap<String, String>) -> usize {
let mut level = 0;
let mut cursor = block_id;
while let Some(parent) = parent_lookup.get(cursor) {
level += 1;
if parent == root_id {
break;
}
cursor = parent;
}
level
}
pub(super) fn text_content(block: &Map, key: &str) -> Option<(String, usize)> {
block.get(key).and_then(|value| {
value.to_text().map(|text| {
let content = text.to_string();
let len = text.len() as usize;
(content, len)
})
})
}
fn determine_display_mode(note_block: Option<&Map>) -> String {
match note_block.and_then(|block| get_string(block, "prop:displayMode")) {
Some(mode) if mode == "both" => "page".into(),
Some(mode) => mode,
None => "edgeless".into(),
}
}
fn compose_additional(
display_mode: &str,
note_block_id: Option<&String>,
database_name: Option<&String>,
) -> Option<String> {
let mut payload = JsonMap::new();
payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string()));
if let Some(note_id) = note_block_id {
payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone()));
}
if let Some(name) = database_name {
payload.insert("databaseName".into(), JsonValue::String(name.clone()));
}
Some(JsonValue::Object(payload).to_string())
}
fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) {
info.blob = Some(vec![blob_id]);
info.content = Some(vec![content]);
}
fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option<String>) {
info.ref_doc_id = Some(vec![page_id]);
if let Some(payload) = payload {
info.ref_info = Some(vec![payload]);
}
}
fn embed_ref_payload(block: &Map, page_id: &str) -> Option<String> {
let params = block.get("prop:params").as_ref().and_then(params_value_to_json);
Some(build_reference_payload(page_id, params))
}
fn gather_surface_texts(block: &Map) -> Vec<String> {
let mut texts = Vec::new();
let elements = match block.get("prop:elements").and_then(|value| value.to_map()) {
Some(map) => map,
None => return texts,
};
if elements
.get("type")
.and_then(|value| value_to_string(&value))
.as_deref()
!= Some("$blocksuite:internal:native$")
{
return texts;
}
if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) {
for value in value_map.values() {
if let Some(element) = value.to_map()
&& let Some(text) = element.get("text").and_then(|value| value.to_text())
{
texts.push(text.to_string());
}
}
}
texts.sort();
texts
}
fn table_cell_texts(block: &Map) -> Vec<String> {
let mut contents = Vec::new();
for key in block.keys() {
if key.starts_with("prop:cells.")
&& key.ends_with(".text")
&& let Some(value) = block.get(key).and_then(|value| value_to_string(&value))
&& !value.is_empty()
{
contents.push(value);
}
}
contents
}
pub(super) fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> {
if let Some((text, len)) = text_content(block, key) {
return Some((text, len));
}
block.get(key).and_then(|value| {
value_to_string(&value).map(|text| {
let len = text.chars().count();
(text, len)
})
})
}
#[cfg(test)]
mod tests {
use serde_json::json;
use y_octo::{AHashMap, Any, DocOptions, TextAttributes, TextDeltaOp, TextInsert, Value};
use super::*;
use crate::doc_parser::build_full_doc;
#[test]
fn test_parse_doc_from_binary() {
let json = include_bytes!("../../../fixtures/demo.ydoc.json");
let doc_bin = include_bytes!("../../../fixtures/demo.ydoc").to_vec();
let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string();
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict)
.numeric_mode(assert_json_diff::NumericMode::AssumeFloat);
assert_json_diff::assert_json_matches!(
serde_json::from_slice::<serde_json::Value>(json).unwrap(),
serde_json::json!(result),
config
);
}
#[test]
fn test_database_cell_references() {
let doc_id = "doc-with-db".to_string();
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
let mut blocks = doc.get_or_create_map("blocks").unwrap();
let mut page = doc.create_map().unwrap();
page.insert("sys:id".into(), "page").unwrap();
page.insert("sys:flavour".into(), "affine:page").unwrap();
let mut page_children = doc.create_array().unwrap();
page_children.push("note").unwrap();
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
let mut page_title = doc.create_text().unwrap();
page_title.insert(0, "Page").unwrap();
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
blocks.insert("page".into(), Value::Map(page)).unwrap();
let mut note = doc.create_map().unwrap();
note.insert("sys:id".into(), "note").unwrap();
note.insert("sys:flavour".into(), "affine:note").unwrap();
let mut note_children = doc.create_array().unwrap();
note_children.push("db").unwrap();
note.insert("sys:children".into(), Value::Array(note_children)).unwrap();
note.insert("prop:displayMode".into(), "page").unwrap();
blocks.insert("note".into(), Value::Map(note)).unwrap();
let mut db = doc.create_map().unwrap();
db.insert("sys:id".into(), "db").unwrap();
db.insert("sys:flavour".into(), "affine:database").unwrap();
db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
.unwrap();
let mut db_title = doc.create_text().unwrap();
db_title.insert(0, "Database").unwrap();
db.insert("prop:title".into(), Value::Text(db_title)).unwrap();
let mut columns = doc.create_array().unwrap();
let mut column = doc.create_map().unwrap();
column.insert("id".into(), "col1").unwrap();
column.insert("name".into(), "Text").unwrap();
column.insert("type".into(), "rich-text").unwrap();
column
.insert("data".into(), Value::Map(doc.create_map().unwrap()))
.unwrap();
columns.push(Value::Map(column)).unwrap();
db.insert("prop:columns".into(), Value::Array(columns)).unwrap();
let mut cell_text = doc.create_text().unwrap();
let mut reference = AHashMap::default();
reference.insert("pageId".into(), Any::String("target-doc".into()));
let mut params = AHashMap::default();
params.insert("mode".into(), Any::String("page".into()));
reference.insert("params".into(), Any::Object(params));
let mut attrs = TextAttributes::new();
attrs.insert("reference".into(), Any::Object(reference));
cell_text
.apply_delta(&[
TextDeltaOp::Insert {
insert: TextInsert::Text("See ".into()),
format: None,
},
TextDeltaOp::Insert {
insert: TextInsert::Text("Target".into()),
format: Some(attrs),
},
])
.unwrap();
let mut cell = doc.create_map().unwrap();
cell.insert("columnId".into(), "col1").unwrap();
cell.insert("value".into(), Value::Text(cell_text)).unwrap();
let mut row = doc.create_map().unwrap();
row.insert("col1".into(), Value::Map(cell)).unwrap();
let mut cells = doc.create_map().unwrap();
cells.insert("row1".into(), Value::Map(row)).unwrap();
db.insert("prop:cells".into(), Value::Map(cells)).unwrap();
blocks.insert("db".into(), Value::Map(db)).unwrap();
let doc_bin = doc.encode_update_v1().unwrap();
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap();
assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()]));
assert_eq!(
db_block.ref_info,
Some(vec![build_reference_payload(
"target-doc",
Some(json!({"mode": "page"}))
)])
);
}
#[test]
fn test_parse_doc_to_markdown_ai_editable_image_table() {
let doc_id = "ai-editable-doc";
let markdown = "![Alt](blob://image-id)\n\n| A | B |\n| --- | --- |\n| 1 | 2 |";
let doc_bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
let result = parse_doc_to_markdown(doc_bin, doc_id.to_string(), true, None).expect("parse doc");
let md = result.markdown;
assert!(md.contains("flavour=affine:image"));
assert!(md.contains("blob://image-id"));
assert!(md.contains("|A|B|"));
assert!(md.contains("|---|---|"));
assert!(
result
.known_unsupported_blocks
.iter()
.any(|block| block.ends_with(":affine:note"))
);
assert!(result.unknown_blocks.is_empty());
}
#[test]
fn test_parse_doc_to_markdown_with_edgeless_text_container() {
let doc_id = "edgeless-text-doc".to_string();
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
let mut blocks = doc.get_or_create_map("blocks").unwrap();
let mut page = doc.create_map().unwrap();
page.insert("sys:id".into(), "page").unwrap();
page.insert("sys:flavour".into(), "affine:page").unwrap();
let mut page_children = doc.create_array().unwrap();
page_children.push("surface").unwrap();
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
let mut page_title = doc.create_text().unwrap();
page_title.insert(0, "Page").unwrap();
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
blocks.insert("page".into(), Value::Map(page)).unwrap();
let mut surface = doc.create_map().unwrap();
surface.insert("sys:id".into(), "surface").unwrap();
surface.insert("sys:flavour".into(), "affine:surface").unwrap();
let mut surface_children = doc.create_array().unwrap();
surface_children.push("edgeless-text").unwrap();
surface
.insert("sys:children".into(), Value::Array(surface_children))
.unwrap();
blocks.insert("surface".into(), Value::Map(surface)).unwrap();
let mut edgeless_text = doc.create_map().unwrap();
edgeless_text.insert("sys:id".into(), "edgeless-text").unwrap();
edgeless_text
.insert("sys:flavour".into(), "affine:edgeless-text")
.unwrap();
let mut edgeless_text_children = doc.create_array().unwrap();
edgeless_text_children.push("paragraph").unwrap();
edgeless_text
.insert("sys:children".into(), Value::Array(edgeless_text_children))
.unwrap();
blocks
.insert("edgeless-text".into(), Value::Map(edgeless_text))
.unwrap();
let mut paragraph = doc.create_map().unwrap();
paragraph.insert("sys:id".into(), "paragraph").unwrap();
paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap();
paragraph
.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
.unwrap();
paragraph.insert("prop:type".into(), "text").unwrap();
let mut paragraph_text = doc.create_text().unwrap();
paragraph_text.insert(0, "hello from edgeless").unwrap();
paragraph
.insert("prop:text".into(), Value::Text(paragraph_text))
.unwrap();
blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap();
let doc_bin = doc.encode_update_v1().unwrap();
let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc");
assert!(result.markdown.is_empty());
assert!(
result
.known_unsupported_blocks
.contains(&"surface:affine:surface".to_string())
);
assert!(
result
.known_unsupported_blocks
.contains(&"edgeless-text:affine:edgeless-text".to_string())
);
assert!(result.unknown_blocks.is_empty());
}
#[test]
fn test_parse_doc_to_markdown_collects_unknown_blocks() {
let doc_id = "unknown-block-doc".to_string();
let doc = DocOptions::new().with_guid(doc_id.clone()).build();
let mut blocks = doc.get_or_create_map("blocks").unwrap();
let mut page = doc.create_map().unwrap();
page.insert("sys:id".into(), "page").unwrap();
page.insert("sys:flavour".into(), "affine:page").unwrap();
let mut page_children = doc.create_array().unwrap();
page_children.push("mystery").unwrap();
page.insert("sys:children".into(), Value::Array(page_children)).unwrap();
let mut page_title = doc.create_text().unwrap();
page_title.insert(0, "Page").unwrap();
page.insert("prop:title".into(), Value::Text(page_title)).unwrap();
blocks.insert("page".into(), Value::Map(page)).unwrap();
let mut mystery = doc.create_map().unwrap();
mystery.insert("sys:id".into(), "mystery").unwrap();
mystery.insert("sys:flavour".into(), "affine:custom-unknown").unwrap();
let mut mystery_children = doc.create_array().unwrap();
mystery_children.push("paragraph").unwrap();
mystery
.insert("sys:children".into(), Value::Array(mystery_children))
.unwrap();
blocks.insert("mystery".into(), Value::Map(mystery)).unwrap();
let mut paragraph = doc.create_map().unwrap();
paragraph.insert("sys:id".into(), "paragraph").unwrap();
paragraph.insert("sys:flavour".into(), "affine:paragraph").unwrap();
paragraph
.insert("sys:children".into(), Value::Array(doc.create_array().unwrap()))
.unwrap();
paragraph.insert("prop:type".into(), "text").unwrap();
let mut paragraph_text = doc.create_text().unwrap();
paragraph_text.insert(0, "child of unknown").unwrap();
paragraph
.insert("prop:text".into(), Value::Text(paragraph_text))
.unwrap();
blocks.insert("paragraph".into(), Value::Map(paragraph)).unwrap();
let doc_bin = doc.encode_update_v1().unwrap();
let result = parse_doc_to_markdown(doc_bin, doc_id, false, None).expect("parse doc");
assert!(result.markdown.is_empty());
assert!(result.known_unsupported_blocks.is_empty());
assert_eq!(result.unknown_blocks, vec!["mystery:affine:custom-unknown".to_string()]);
}
}
packages/common/native/src/doc_parser/roundtrip_tests.rs
-56
View File
@@ -1,56 +0,0 @@
use super::{build_full_doc, parse_doc_to_markdown};
fn assert_markdown_roundtrip(markdown: &str, expected: &str) {
let doc_id = "roundtrip-doc";
let title = "Roundtrip Title";
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
assert_eq!(result.title, title);
assert_eq!(result.markdown, expected);
}
#[test]
fn test_roundtrip_inline_styles() {
let markdown = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).";
let expected = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).\n\n";
assert_markdown_roundtrip(markdown, expected);
}
#[test]
fn test_roundtrip_list_items() {
let markdown = "- Item 1\n- Item 2\n- [ ] Task\n- [x] Done";
let expected = "* Item 1\n* Item 2\n- [ ] Task\n- [x] Done\n";
assert_markdown_roundtrip(markdown, expected);
}
#[test]
fn test_roundtrip_code_block() {
let markdown = "```rust\nfn main() {}\n```";
let expected = "```rust\nfn main() {}\n\n```\n\n";
assert_markdown_roundtrip(markdown, expected);
}
#[test]
fn test_roundtrip_code_block_indentation() {
let markdown = "```python\n def indented():\n return \"ok\"\n```";
let doc_id = "roundtrip-indent";
let title = "Roundtrip Title";
let bin = build_full_doc(title, markdown, doc_id).expect("create doc");
let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc");
assert!(result.markdown.contains("\n def indented():"));
assert!(result.markdown.contains("\n return \"ok\""));
}
#[test]
fn test_roundtrip_table() {
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
let expected = "|A|B|\n|---|---|\n|1|2|\n\n";
assert_markdown_roundtrip(markdown, expected);
}
#[test]
fn test_roundtrip_image_with_caption() {
let markdown = "![Alt](blob://image-id)";
let expected = "<img\n src=\"blob://image-id\"\n alt=\"Alt\"\n width=\"auto\"\n height=\"auto\"\n/>\n\n";
assert_markdown_roundtrip(markdown, expected);
}
packages/common/native/src/doc_parser/schema.rs
-57
View File
@@ -1,57 +0,0 @@
pub(super) const PAGE_FLAVOUR: &str = "affine:page";
pub(super) const NOTE_FLAVOUR: &str = "affine:note";
pub(super) const SURFACE_FLAVOUR: &str = "affine:surface";
pub(super) const SYS_ID: &str = "sys:id";
pub(super) const SYS_FLAVOUR: &str = "sys:flavour";
pub(super) const SYS_VERSION: &str = "sys:version";
pub(super) const SYS_CHILDREN: &str = "sys:children";
pub(super) const PROP_TITLE: &str = "prop:title";
pub(super) const PROP_TEXT: &str = "prop:text";
pub(super) const PROP_TYPE: &str = "prop:type";
pub(super) const PROP_CHECKED: &str = "prop:checked";
pub(super) const PROP_LANGUAGE: &str = "prop:language";
pub(super) const PROP_ORDER: &str = "prop:order";
pub(super) const PROP_ELEMENTS: &str = "prop:elements";
pub(super) const PROP_BACKGROUND: &str = "prop:background";
pub(super) const PROP_XYWH: &str = "prop:xywh";
pub(super) const PROP_INDEX: &str = "prop:index";
pub(super) const PROP_HIDDEN: &str = "prop:hidden";
pub(super) const PROP_DISPLAY_MODE: &str = "prop:displayMode";
pub(super) const PROP_SOURCE_ID: &str = "prop:sourceId";
pub(super) const PROP_CAPTION: &str = "prop:caption";
pub(super) const PROP_WIDTH: &str = "prop:width";
pub(super) const PROP_HEIGHT: &str = "prop:height";
pub(super) const PROP_URL: &str = "prop:url";
pub(super) const PROP_VIDEO_ID: &str = "prop:videoId";
pub(super) const PROP_ROWS_PREFIX: &str = "prop:rows.";
pub(super) const PROP_COLUMNS_PREFIX: &str = "prop:columns.";
pub(super) const PROP_CELLS_PREFIX: &str = "prop:cells.";
pub(super) const PROP_ROW_ID_SUFFIX: &str = ".rowId";
pub(super) const PROP_COLUMN_ID_SUFFIX: &str = ".columnId";
pub(super) const PROP_ORDER_SUFFIX: &str = ".order";
pub(super) const PROP_TEXT_SUFFIX: &str = ".text";
pub(super) fn table_row_id_key(row_id: &str) -> String {
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ROW_ID_SUFFIX}")
}
pub(super) fn table_row_order_key(row_id: &str) -> String {
format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ORDER_SUFFIX}")
}
pub(super) fn table_column_id_key(column_id: &str) -> String {
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_COLUMN_ID_SUFFIX}")
}
pub(super) fn table_column_order_key(column_id: &str) -> String {
format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_ORDER_SUFFIX}")
}
pub(super) fn table_cell_text_key(row_id: &str, column_id: &str) -> String {
format!("{PROP_CELLS_PREFIX}{row_id}:{column_id}{PROP_TEXT_SUFFIX}")
}
packages/common/native/src/doc_parser/table.rs
-71
View File
@@ -1,71 +0,0 @@
#[derive(Clone, Copy)]
pub(super) struct MarkdownTableOptions {
pub(super) escape_pipes: bool,
pub(super) newline_replacement: &'static str,
pub(super) trim: bool,
}
impl MarkdownTableOptions {
pub(super) const fn new(escape_pipes: bool, newline_replacement: &'static str, trim: bool) -> Self {
Self {
escape_pipes,
newline_replacement,
trim,
}
}
}
pub(super) fn render_markdown_table(rows: &[Vec<String>], options: MarkdownTableOptions) -> Option<String> {
let (header, body) = rows.split_first()?;
let header_line = format_table_row(header, options);
let separator_line = format_table_row(&vec!["---".to_string(); header.len()], options);
let mut lines = vec![header_line, separator_line];
for row in body {
lines.push(format_table_row(row, options));
}
Some(lines.join("\n"))
}
fn format_table_row(row: &[String], options: MarkdownTableOptions) -> String {
let cells = row
.iter()
.map(|cell| format_table_cell(cell, options))
.collect::<Vec<_>>();
format!("|{}|", cells.join("|"))
}
fn format_table_cell(cell: &str, options: MarkdownTableOptions) -> String {
let mut value = if options.trim {
cell.trim().to_string()
} else {
cell.to_string()
};
if options.escape_pipes {
value = value.replace('|', "\\|");
}
if !options.newline_replacement.is_empty() {
value = collapse_newlines(&value, options.newline_replacement);
}
value
}
fn collapse_newlines(value: &str, replacement: &str) -> String {
if replacement.is_empty() {
return value.to_string();
}
let mut out = String::with_capacity(value.len());
let mut in_newline = false;
for ch in value.chars() {
if ch == '\n' {
if !in_newline {
out.push_str(replacement);
in_newline = true;
}
} else {
in_newline = false;
out.push(ch);
}
}
out
}
packages/common/native/src/doc_parser/value.rs
-131
View File
@@ -1,131 +0,0 @@
use serde_json::{Map as JsonMap, Value as JsonValue};
use y_octo::{AHashMap, Any, Value};
pub(super) fn any_truthy(value: &Any) -> bool {
match value {
Any::True => true,
Any::False | Any::Null | Any::Undefined => false,
Any::String(value) => !value.is_empty(),
Any::Integer(value) => *value != 0,
Any::Float32(value) => value.0 != 0.0,
Any::Float64(value) => value.0 != 0.0,
Any::BigInt64(value) => *value != 0,
Any::Object(_) | Any::Array(_) | Any::Binary(_) => true,
}
}
pub(super) fn any_as_string(value: &Any) -> Option<&str> {
match value {
Any::String(value) => Some(value),
_ => None,
}
}
pub(super) fn any_as_u64(value: &Any) -> Option<u64> {
match value {
Any::Integer(value) if *value >= 0 => Some(*value as u64),
Any::Float32(value) if value.0 >= 0.0 => Some(value.0 as u64),
Any::Float64(value) if value.0 >= 0.0 => Some(value.0 as u64),
Any::BigInt64(value) if *value >= 0 => Some(*value as u64),
_ => None,
}
}
pub(super) fn value_to_string(value: &Value) -> Option<String> {
if let Some(text) = value.to_text() {
return Some(text.to_string());
}
if let Some(any) = value.to_any() {
return any_to_string(&any);
}
None
}
pub(super) fn value_to_any(value: &Value) -> Option<Any> {
if let Some(any) = value.to_any() {
return Some(any);
}
if let Some(text) = value.to_text() {
return Some(Any::String(text.to_string()));
}
if let Some(array) = value.to_array() {
let mut values = Vec::new();
for item in array.iter() {
if let Some(any) = value_to_any(&item) {
values.push(any);
} else if let Some(text) = value_to_string(&item) {
values.push(Any::String(text));
}
}
return Some(Any::Array(values));
}
if let Some(map) = value.to_map() {
let mut values = AHashMap::default();
for key in map.keys() {
if let Some(entry) = map.get(key) {
if let Some(any) = value_to_any(&entry) {
values.insert(key.to_string(), any);
} else if let Some(text) = value_to_string(&entry) {
values.insert(key.to_string(), Any::String(text));
}
}
}
return Some(Any::Object(values));
}
None
}
pub(super) fn value_to_f64(value: Value) -> Option<f64> {
value.to_any().and_then(|any| match any {
Any::Integer(v) => Some(v as f64),
Any::BigInt64(v) => Some(v as f64),
Any::Float32(v) => Some(v.0 as f64),
Any::Float64(v) => Some(v.0),
_ => None,
})
}
pub(super) fn any_to_string(any: &Any) -> Option<String> {
match any {
Any::String(value) => Some(value.to_string()),
Any::Integer(value) => Some(value.to_string()),
Any::Float32(value) => Some(value.0.to_string()),
Any::Float64(value) => Some(value.0.to_string()),
Any::BigInt64(value) => Some(value.to_string()),
Any::True => Some("true".into()),
Any::False => Some("false".into()),
Any::Null | Any::Undefined => None,
Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(),
}
}
pub(super) fn params_any_map_to_json(params: &AHashMap<String, Any>) -> JsonValue {
let mut values = JsonMap::new();
for (key, value) in params.iter() {
if let Ok(value) = serde_json::to_value(value) {
values.insert(key.clone(), value);
}
}
JsonValue::Object(values)
}
pub(super) fn params_value_to_json(params: &Value) -> Option<JsonValue> {
serde_json::to_value(params).ok()
}
pub(super) fn build_reference_payload(doc_id: &str, params: Option<JsonValue>) -> String {
let mut payload = JsonMap::new();
payload.insert("docId".into(), JsonValue::String(doc_id.to_string()));
if let Some(JsonValue::Object(params)) = params {
for (key, value) in params.into_iter() {
payload.insert(key, value);
}
}
JsonValue::Object(payload).to_string()
}
packages/common/native/src/doc_parser/write/builder.rs
-426
View File
@@ -1,426 +0,0 @@
use y_octo::{TextDeltaOp, TextInsert};
use super::{
super::schema::{
PROP_CAPTION, PROP_CELLS_PREFIX, PROP_CHECKED, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER,
PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, PROP_VIDEO_ID, PROP_WIDTH, SYS_CHILDREN,
SYS_FLAVOUR, SYS_ID, SYS_VERSION, table_cell_text_key, table_column_id_key, table_column_order_key,
table_row_id_key, table_row_order_key,
},
*,
};
pub(super) const BOXED_NATIVE_TYPE: &str = "$blocksuite:internal:native$";
pub(super) const NOTE_BG_LIGHT: &str = "#ffffff";
pub(super) const NOTE_BG_DARK: &str = "#252525";
const TABLE_ORDER_WIDTH: usize = 6;
pub(super) fn block_version(flavour: &str) -> i32 {
match flavour {
"affine:page" => 2,
"affine:surface" => 5,
"affine:note" => 1,
"affine:paragraph" => 1,
"affine:list" => 1,
"affine:code" => 1,
"affine:divider" => 1,
"affine:image" => 1,
"affine:table" => 1,
"affine:bookmark" => 1,
"affine:embed-youtube" => 1,
"affine:embed-iframe" => 1,
"affine:callout" => 1,
_ => 1,
}
}
pub(super) struct TextBlockProps<'a> {
pub block_type: Option<&'a str>,
pub checked: Option<bool>,
pub language: Option<&'a str>,
pub order: Option<i64>,
pub text: &'a [TextDeltaOp],
}
pub(super) struct ImageBlockProps<'a> {
pub source_id: &'a str,
pub caption: Option<&'a str>,
pub width: Option<f64>,
pub height: Option<f64>,
}
pub(super) struct BookmarkBlockProps<'a> {
pub url: &'a str,
pub caption: Option<&'a str>,
}
pub(super) struct EmbedYoutubeBlockProps<'a> {
pub video_id: &'a str,
}
pub(super) struct EmbedIframeBlockProps<'a> {
pub url: &'a str,
}
pub(super) fn insert_text(doc: &Doc, block: &mut Map, key: &str, ops: &[TextDeltaOp]) -> Result<(), ParseError> {
let mut text = doc.create_text()?;
// Attach first so updates encode parent types before their contents.
block.insert(key.to_string(), Value::Text(text.clone()))?;
if !ops.is_empty() {
text.apply_delta(ops)?;
}
Ok(())
}
pub(crate) fn text_ops_from_plain(text: &str) -> Vec<TextDeltaOp> {
if text.is_empty() {
Vec::new()
} else {
vec![TextDeltaOp::Insert {
insert: TextInsert::Text(text.to_string()),
format: None,
}]
}
}
pub(super) fn insert_children(doc: &Doc, block: &mut Map, children: &[String]) -> Result<(), ParseError> {
let mut array = doc.create_array()?;
// Attach first so updates encode parent types before their contents.
block.insert(SYS_CHILDREN.to_string(), Value::Array(array.clone()))?;
for child_id in children {
array.push(child_id.to_string())?;
}
Ok(())
}
pub(super) fn insert_block_map(doc: &Doc, blocks_map: &mut Map, block_id: &str) -> Result<Map, ParseError> {
let empty_map = doc.create_map()?;
blocks_map.insert(block_id.to_string(), Value::Map(empty_map))?;
blocks_map
.get(block_id)
.and_then(|value| value.to_map())
.ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted block map".into()))
}
pub(super) fn insert_sys_fields(block: &mut Map, block_id: &str, flavour: &str) -> Result<(), ParseError> {
block.insert(SYS_ID.to_string(), Any::String(block_id.to_string()))?;
block.insert(SYS_FLAVOUR.to_string(), Any::String(flavour.to_string()))?;
block.insert(SYS_VERSION.to_string(), Any::Integer(block_version(flavour)))?;
Ok(())
}
pub(super) fn apply_text_block_props(
doc: &Doc,
block: &mut Map,
props: &TextBlockProps<'_>,
preserve_text: bool,
clear_missing: bool,
) -> Result<(), ParseError> {
match props.block_type {
Some(block_type) => {
block.insert(PROP_TYPE.to_string(), Any::String(block_type.to_string()))?;
}
None => {
if clear_missing && block.get(PROP_TYPE).is_some() {
block.remove(PROP_TYPE);
}
}
}
if !preserve_text && !props.text.is_empty() {
insert_text(doc, block, PROP_TEXT, props.text)?;
} else if !preserve_text && clear_missing && block.get(PROP_TEXT).is_some() {
block.remove(PROP_TEXT);
}
match props.checked {
Some(checked) => {
block.insert(PROP_CHECKED.to_string(), if checked { Any::True } else { Any::False })?;
}
None => {
if clear_missing && block.get(PROP_CHECKED).is_some() {
block.remove(PROP_CHECKED);
}
}
}
match props.language {
Some(language) => {
block.insert(PROP_LANGUAGE.to_string(), Any::String(language.to_string()))?;
}
None => {
if clear_missing && block.get(PROP_LANGUAGE).is_some() {
block.remove(PROP_LANGUAGE);
}
}
}
match props.order {
Some(order) => {
block.insert(PROP_ORDER.to_string(), Any::Float64((order as f64).into()))?;
}
None => {
if clear_missing && block.get(PROP_ORDER).is_some() {
block.remove(PROP_ORDER);
}
}
}
Ok(())
}
pub(super) fn apply_image_block_props(
block: &mut Map,
props: &ImageBlockProps<'_>,
clear_missing: bool,
) -> Result<(), ParseError> {
block.insert(PROP_SOURCE_ID.to_string(), Any::String(props.source_id.to_string()))?;
match props.caption {
Some(caption) => {
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
}
None => {
if clear_missing && block.get(PROP_CAPTION).is_some() {
block.remove(PROP_CAPTION);
}
}
}
match props.width {
Some(width) => {
block.insert(PROP_WIDTH.to_string(), Any::Float64(width.into()))?;
}
None => {
if clear_missing && block.get(PROP_WIDTH).is_some() {
block.remove(PROP_WIDTH);
}
}
}
match props.height {
Some(height) => {
block.insert(PROP_HEIGHT.to_string(), Any::Float64(height.into()))?;
}
None => {
if clear_missing && block.get(PROP_HEIGHT).is_some() {
block.remove(PROP_HEIGHT);
}
}
}
Ok(())
}
pub(super) fn apply_bookmark_block_props(
block: &mut Map,
props: &BookmarkBlockProps<'_>,
clear_missing: bool,
) -> Result<(), ParseError> {
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
match props.caption {
Some(caption) => {
block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?;
}
None => {
if clear_missing && block.get(PROP_CAPTION).is_some() {
block.remove(PROP_CAPTION);
}
}
}
Ok(())
}
pub(super) fn apply_embed_youtube_block_props(
block: &mut Map,
props: &EmbedYoutubeBlockProps<'_>,
) -> Result<(), ParseError> {
block.insert(PROP_VIDEO_ID.to_string(), Any::String(props.video_id.to_string()))?;
Ok(())
}
pub(super) fn apply_embed_iframe_block_props(
block: &mut Map,
props: &EmbedIframeBlockProps<'_>,
) -> Result<(), ParseError> {
block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?;
Ok(())
}
pub(super) fn apply_table_block_props(block: &mut Map, rows: &[Vec<String>]) -> Result<(), ParseError> {
clear_table_props(block);
if rows.is_empty() {
return Ok(());
}
let column_count = rows.iter().map(|row| row.len()).max().unwrap_or(0);
let column_ids: Vec<String> = (0..column_count).map(|_| nanoid::nanoid!()).collect();
for (col_idx, column_id) in column_ids.iter().enumerate() {
let order = format_table_order(col_idx);
block.insert(table_column_id_key(column_id), Any::String(column_id.to_string()))?;
block.insert(table_column_order_key(column_id), Any::String(order))?;
}
for (row_idx, row) in rows.iter().enumerate() {
let row_id = nanoid::nanoid!();
let order = format_table_order(row_idx);
block.insert(table_row_id_key(&row_id), Any::String(row_id.to_string()))?;
block.insert(table_row_order_key(&row_id), Any::String(order))?;
for (col_idx, column_id) in column_ids.iter().enumerate() {
let cell_text = row.get(col_idx).cloned().unwrap_or_default();
block.insert(table_cell_text_key(&row_id, column_id), Any::String(cell_text))?;
}
}
Ok(())
}
pub(super) struct ApplyBlockOptions {
pub preserve_text: bool,
pub clear_missing: bool,
}
pub(super) fn apply_block_spec(
doc: &Doc,
block: &mut Map,
spec: &BlockSpec,
options: ApplyBlockOptions,
) -> Result<(), ParseError> {
match spec.flavour {
BlockFlavour::Image => {
if options.preserve_text {
return Ok(());
}
let image = spec
.image
.as_ref()
.ok_or_else(|| ParseError::ParserError("image spec missing".into()))?;
let props = ImageBlockProps {
source_id: &image.source_id,
caption: image.caption.as_deref(),
width: image.width,
height: image.height,
};
apply_image_block_props(block, &props, options.clear_missing)?;
}
BlockFlavour::Bookmark => {
if options.preserve_text {
return Ok(());
}
let bookmark = spec
.bookmark
.as_ref()
.ok_or_else(|| ParseError::ParserError("bookmark spec missing".into()))?;
let props = BookmarkBlockProps {
url: &bookmark.url,
caption: bookmark.caption.as_deref(),
};
apply_bookmark_block_props(block, &props, options.clear_missing)?;
}
BlockFlavour::EmbedYoutube => {
if options.preserve_text {
return Ok(());
}
let embed = spec
.embed_youtube
.as_ref()
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
let props = EmbedYoutubeBlockProps {
video_id: &embed.video_id,
};
apply_embed_youtube_block_props(block, &props)?;
}
BlockFlavour::EmbedIframe => {
if options.preserve_text {
return Ok(());
}
let embed = spec
.embed_iframe
.as_ref()
.ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?;
let props = EmbedIframeBlockProps { url: &embed.url };
apply_embed_iframe_block_props(block, &props)?;
}
BlockFlavour::Callout => {
return Ok(());
}
BlockFlavour::Table => {
if options.preserve_text {
return Ok(());
}
let table = spec
.table
.as_ref()
.ok_or_else(|| ParseError::ParserError("table spec missing".into()))?;
apply_table_block_props(block, &table.rows)?;
}
_ => {
let props = TextBlockProps {
block_type: spec.block_type_str(),
checked: spec.checked,
language: spec.language.as_deref(),
order: spec.order,
text: &spec.text,
};
apply_text_block_props(doc, block, &props, options.preserve_text, options.clear_missing)?;
}
}
Ok(())
}
pub(super) fn insert_block_tree(doc: &Doc, blocks_map: &mut Map, node: &BlockNode) -> Result<String, ParseError> {
let block_id = nanoid::nanoid!();
let mut block_map = insert_block_map(doc, blocks_map, &block_id)?;
insert_sys_fields(&mut block_map, &block_id, node.spec.flavour.as_str())?;
apply_block_spec(
doc,
&mut block_map,
&node.spec,
ApplyBlockOptions {
preserve_text: false,
clear_missing: false,
},
)?;
let child_ids = node
.children
.iter()
.map(|child| insert_block_tree(doc, blocks_map, child))
.collect::<Result<Vec<_>, _>>()?;
insert_children(doc, &mut block_map, &child_ids)?;
Ok(block_id)
}
fn clear_table_props(block: &mut Map) {
let keys = block
.keys()
.filter(|key| {
key.starts_with(PROP_ROWS_PREFIX) || key.starts_with(PROP_COLUMNS_PREFIX) || key.starts_with(PROP_CELLS_PREFIX)
})
.map(|s| s.to_string())
.collect::<Vec<_>>();
for key in keys {
block.remove(&key);
}
}
fn format_table_order(index: usize) -> String {
format!("{index:0width$}", width = TABLE_ORDER_WIDTH)
}
pub(super) fn boxed_empty_map(doc: &Doc) -> Result<Map, ParseError> {
doc.create_map().map_err(ParseError::from)
}
pub(super) fn note_background_map(doc: &Doc) -> Result<Map, ParseError> {
doc.create_map().map_err(ParseError::from)
}
packages/common/native/src/doc_parser/write/create.rs
-281
View File
@@ -1,281 +0,0 @@
//! Markdown to YDoc conversion module
//!
//! Converts markdown content into AFFiNE-compatible y-octo document binary
//! format.
use y_octo::DocOptions;
use super::{
super::{
markdown::parse_markdown_blocks,
schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR},
},
builder::{
BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, boxed_empty_map, insert_block_map, insert_block_tree,
insert_children, insert_sys_fields, insert_text, note_background_map, text_ops_from_plain,
},
*,
};
/// Converts markdown into an AFFiNE-compatible y-octo document binary.
///
/// # Arguments
/// * `title` - The document title
/// * `markdown` - The markdown content to convert
/// * `doc_id` - The document ID to use
///
/// # Returns
/// A binary vector containing the y-octo encoded document update
pub fn build_full_doc(title: &str, markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
let nodes = parse_markdown_blocks(markdown)?;
build_doc_update(doc_id, title, &nodes)
}
fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result<Vec<u8>, ParseError> {
let doc = DocOptions::new().with_guid(doc_id.to_string()).build();
let mut blocks_map = doc.get_or_create_map("blocks")?;
let page_id = nanoid::nanoid!();
let surface_id = nanoid::nanoid!();
let note_id = nanoid::nanoid!();
// Insert root blocks first to establish stable IDs.
let mut page_map = insert_block_map(&doc, &mut blocks_map, &page_id)?;
let mut surface_map = insert_block_map(&doc, &mut blocks_map, &surface_id)?;
let mut note_map = insert_block_map(&doc, &mut blocks_map, &note_id)?;
// Create content blocks under note.
let content_ids = insert_block_trees(&doc, &mut blocks_map, blocks)?;
// Page block
insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?;
insert_children(&doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?;
insert_text(&doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?;
// Surface block
insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?;
insert_children(&doc, &mut surface_map, &[])?;
let mut boxed = boxed_empty_map(&doc)?;
surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?;
boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?;
let value = doc.create_map()?;
boxed.insert("value".to_string(), Value::Map(value))?;
// Note block
insert_sys_fields(&mut note_map, &note_id, NOTE_FLAVOUR)?;
insert_children(&doc, &mut note_map, &content_ids)?;
let mut background = note_background_map(&doc)?;
note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?;
background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?;
background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?;
note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?;
note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?;
note_map.insert(PROP_HIDDEN.to_string(), Any::False)?;
note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?;
Ok(doc.encode_update_v1()?)
}
fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result<Vec<String>, ParseError> {
let mut ids = Vec::with_capacity(blocks.len());
for block in blocks {
let id = insert_block_tree(doc, blocks_map, block)?;
ids.push(id);
}
Ok(ids)
}
#[cfg(test)]
mod tests {
use y_octo::{Any, DocOptions};
use super::{
super::super::{
blocksuite::get_string,
markdown::{MAX_BLOCKS, MAX_MARKDOWN_CHARS},
schema::PAGE_FLAVOUR,
},
*,
};
#[test]
fn test_simple_markdown() {
let markdown = "# Hello World\n\nThis is a test paragraph.";
let result = build_full_doc("Hello World", markdown, "test-doc-id");
assert!(result.is_ok());
let bin = result.unwrap();
assert!(!bin.is_empty());
}
#[test]
fn test_title_from_param() {
let markdown = "# Markdown Title\n\nContent.";
let doc_id = "title-param-test";
let bin = build_full_doc("External Title", markdown, doc_id).expect("create doc");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&bin).expect("apply update");
let blocks_map = doc.get_map("blocks").expect("blocks map");
let mut title = None;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
{
title = get_string(&block_map, "prop:title");
break;
}
}
assert_eq!(title.as_deref(), Some("External Title"));
}
#[test]
fn test_markdown_with_list() {
let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3";
let result = build_full_doc("Test List", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_markdown_with_code() {
let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```";
let result = build_full_doc("Code Example", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_markdown_with_headings() {
let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text.";
let result = build_full_doc("H1", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_empty_markdown() {
let result = build_full_doc("Untitled", "", "test-doc-id");
assert!(result.is_ok());
let bin = result.unwrap();
assert!(!bin.is_empty());
}
#[test]
fn test_whitespace_only_markdown() {
let result = build_full_doc("Untitled", " \n\n\t\n ", "test-doc-id");
assert!(result.is_ok());
let bin = result.unwrap();
assert!(!bin.is_empty());
}
#[test]
fn test_markdown_without_h1() {
let markdown = "## Secondary Heading\n\nSome content without H1.";
let result = build_full_doc("Title", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_nested_lists() {
let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1";
let result = build_full_doc("Nested Lists", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_blockquote() {
let markdown = "# Title\n\n> A blockquote";
let result = build_full_doc("Title", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_divider() {
let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider";
let result = build_full_doc("Title", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_numbered_list() {
let markdown = "# Title\n\n1. First item\n2. Second item";
let result = build_full_doc("Title", markdown, "test-doc-id");
assert!(result.is_ok());
}
#[test]
fn test_markdown_too_large() {
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
let result = build_full_doc("Title", &markdown, "test-doc-id");
assert!(result.is_err());
}
#[test]
fn test_markdown_block_limit() {
let mut markdown = String::from("# Title\n\n");
for i in 0..=MAX_BLOCKS {
markdown.push_str(&format!("Paragraph {i}\n\n"));
}
let result = build_full_doc("Title", &markdown, "test-doc-id");
assert!(result.is_err());
}
#[test]
fn test_markdown_with_image() {
let markdown = "![Alt](blob://image-id)";
let doc_id = "image-doc";
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&bin).expect("apply update");
let blocks_map = doc.get_map("blocks").expect("blocks map");
let mut found = false;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
{
let source_id = get_string(&block_map, "prop:sourceId");
assert_eq!(source_id.as_deref(), Some("image-id"));
found = true;
break;
}
}
assert!(found);
}
#[test]
fn test_markdown_with_table() {
let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |";
let doc_id = "table-doc";
let bin = build_full_doc("Title", markdown, doc_id).expect("create doc");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&bin).expect("apply update");
let blocks_map = doc.get_map("blocks").expect("blocks map");
let mut found_cell = false;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
{
for key in block_map.keys() {
if key.starts_with("prop:cells.") && key.ends_with(".text") {
let value = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
Any::String(value) => Some(value),
_ => None,
});
if let Some(value) = value
&& (value == "A" || value == "1")
{
found_cell = true;
break;
}
}
}
}
}
assert!(found_cell);
}
}
packages/common/native/src/doc_parser/write/doc_meta.rs
-132
View File
@@ -1,132 +0,0 @@
use super::{
builder::{insert_text, text_ops_from_plain},
root_doc::ensure_pages_array,
*,
};
pub fn update_doc_title(existing_binary: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
let doc = load_doc(existing_binary, Some(doc_id))?;
let state_before = doc.get_state_vector();
let blocks_map = doc.get_map("blocks")?;
if blocks_map.is_empty() {
return Err(ParseError::ParserError("blocks map is empty".into()));
}
let mut page_block = find_page_block(&blocks_map)?;
let current = get_string(&page_block, PROP_TITLE).unwrap_or_default();
if current != title {
insert_text(&doc, &mut page_block, PROP_TITLE, &text_ops_from_plain(title))?;
}
Ok(doc.encode_state_as_update_v1(&state_before)?)
}
pub fn update_root_doc_meta_title(root_doc_bin: &[u8], doc_id: &str, title: &str) -> Result<Vec<u8>, ParseError> {
let doc = load_doc_or_new(root_doc_bin)?;
let state_before = doc.get_state_vector();
let mut meta = doc.get_or_create_map("meta")?;
let mut pages = ensure_pages_array(&doc, &mut meta)?;
let mut found = false;
for idx in 0..pages.len() {
let Some(mut page) = pages.get(idx).and_then(|v| v.to_map()) else {
continue;
};
if get_string(&page, "id").as_deref() == Some(doc_id) {
page.insert("title".to_string(), Any::String(title.to_string()))?;
found = true;
break;
}
}
if !found {
let page_map = doc.create_map()?;
let idx = pages.len();
pages.insert(idx, Value::Map(page_map))?;
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
inserted_page.insert("title".to_string(), Any::String(title.to_string()))?;
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0);
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
let tags = doc.create_array()?;
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
}
}
Ok(doc.encode_state_as_update_v1(&state_before)?)
}
fn find_page_block(blocks_map: &Map) -> Result<Map, ParseError> {
let index = build_block_index(blocks_map);
let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
blocks_map
.get(&page_id)
.and_then(|value| value.to_map())
.ok_or_else(|| ParseError::ParserError("page block not found".into()))
}
#[cfg(test)]
mod tests {
use y_octo::DocOptions;
use super::*;
use crate::doc_parser::{add_doc_to_root_doc, build_full_doc};
#[test]
fn test_update_doc_title() {
let doc_id = "doc-meta-title-test";
let initial = build_full_doc("Old Title", "Content.", doc_id).expect("create doc");
let delta = update_doc_title(&initial, doc_id, "New Title").expect("update title");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&initial).expect("apply initial");
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
let blocks_map = doc.get_map("blocks").expect("blocks map");
let mut title = None;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
{
title = get_string(&block_map, "prop:title");
break;
}
}
assert_eq!(title.as_deref(), Some("New Title"));
}
#[test]
fn test_update_root_doc_meta_title() {
let doc_id = "root-meta-title-test";
let root_bin = add_doc_to_root_doc(Vec::new(), doc_id, Some("Old Title")).expect("create root meta");
let delta = update_root_doc_meta_title(&root_bin, doc_id, "New Title").expect("update meta");
let mut doc = DocOptions::new().build();
doc.apply_update_from_binary_v1(&root_bin).expect("apply root");
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
let meta = doc.get_map("meta").expect("meta map");
let pages = meta.get("pages").and_then(|v| v.to_array()).expect("pages array");
let mut title = None;
for page in pages.iter() {
if let Some(page_map) = page.to_map()
&& get_string(&page_map, "id").as_deref() == Some(doc_id)
{
title = get_string(&page_map, "title");
break;
}
}
assert_eq!(title.as_deref(), Some("New Title"));
}
}
packages/common/native/src/doc_parser/write/doc_properties.rs
-95
View File
@@ -1,95 +0,0 @@
use y_octo::{Any, DocOptions, Map};
use super::{
super::{doc_loader::is_empty_doc, value::value_to_string},
ParseError,
};
pub fn update_doc_properties(
existing_binary: &[u8],
properties_doc_id: &str,
target_doc_id: &str,
created_by: Option<&str>,
updated_by: Option<&str>,
) -> Result<Vec<u8>, ParseError> {
let doc = if is_empty_doc(existing_binary) {
DocOptions::new().with_guid(properties_doc_id.to_string()).build()
} else {
super::load_doc(existing_binary, Some(properties_doc_id))?
};
let state_before = doc.get_state_vector();
let mut record = doc.get_or_create_map(target_doc_id)?;
let mut changed = false;
if record.get("id").is_none() {
record.insert("id".to_string(), Any::String(target_doc_id.to_string()))?;
changed = true;
}
if let Some(created_by) = created_by
&& get_record_string(&record, "createdBy").as_deref() != Some(created_by)
{
record.insert("createdBy".to_string(), Any::String(created_by.to_string()))?;
changed = true;
}
if let Some(updated_by) = updated_by
&& get_record_string(&record, "updatedBy").as_deref() != Some(updated_by)
{
record.insert("updatedBy".to_string(), Any::String(updated_by.to_string()))?;
changed = true;
}
if !changed {
return Ok(Vec::new());
}
Ok(doc.encode_state_as_update_v1(&state_before)?)
}
fn get_record_string(record: &Map, key: &str) -> Option<String> {
record.get(key).and_then(|value| value_to_string(&value))
}
#[cfg(test)]
mod tests {
use y_octo::DocOptions;
use super::*;
#[test]
fn test_update_doc_properties_creates_record() {
let properties_doc_id = "doc-properties";
let target_doc_id = "doc-1";
let update = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-1"))
.expect("update properties");
let mut doc = DocOptions::new().with_guid(properties_doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&update).expect("apply");
let record = doc.get_map(target_doc_id).expect("record map");
assert_eq!(get_record_string(&record, "id").as_deref(), Some(target_doc_id));
assert_eq!(get_record_string(&record, "createdBy").as_deref(), Some("user-1"));
assert_eq!(get_record_string(&record, "updatedBy").as_deref(), Some("user-1"));
}
#[test]
fn test_update_doc_properties_no_change() {
let properties_doc_id = "doc-properties-no-change";
let target_doc_id = "doc-2";
let initial = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-2"))
.expect("initial update");
let delta = update_doc_properties(
&initial,
properties_doc_id,
target_doc_id,
Some("user-1"),
Some("user-2"),
)
.expect("no change update");
assert!(delta.is_empty());
}
}
packages/common/native/src/doc_parser/write/mod.rs
-21
View File
@@ -1,21 +0,0 @@
pub mod builder;
mod create;
mod doc_meta;
mod doc_properties;
mod root_doc;
mod update;
pub use create::build_full_doc;
pub use doc_meta::{update_doc_title, update_root_doc_meta_title};
pub use doc_properties::update_doc_properties;
pub use root_doc::{add_doc_to_root_doc, build_public_root_doc};
pub use update::update_doc;
use y_octo::{Any, Doc, Map, Value};
use super::{
ParseError,
block_spec::{BlockFlavour, BlockNode, BlockSpec},
blocksuite::{build_block_index, find_block_id_by_flavour, get_string},
doc_loader::{load_doc, load_doc_or_new},
schema::{NOTE_FLAVOUR, PAGE_FLAVOUR, PROP_TITLE},
};
packages/common/native/src/doc_parser/write/root_doc.rs
-291
View File
@@ -1,291 +0,0 @@
use std::collections::HashSet;
use y_octo::Array;
use super::*;
const DEFAULT_DOC_TITLE: &str = "Untitled";
const SIMPLE_PAGE_META_KEYS: &[&str] = &[
"id",
"title",
"createDate",
"updatedDate",
"trash",
"trashDate",
"headerImage",
];
fn any_to_value(doc: &Doc, any: Any) -> Result<Value, ParseError> {
match any {
Any::Array(values) => {
let mut array = doc.create_array()?;
for value in values {
let item = any_to_value(doc, value)?;
array.push(item)?;
}
Ok(Value::Array(array))
}
Any::Object(values) => {
let mut map = doc.create_map()?;
for (key, value) in values {
let item = any_to_value(doc, value)?;
map.insert(key, item)?;
}
Ok(Value::Map(map))
}
_ => Ok(Value::Any(any)),
}
}
pub(super) fn ensure_pages_array(doc: &Doc, meta: &mut Map) -> Result<Array, ParseError> {
let pages_value = meta.get("pages");
if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) {
return Ok(pages);
}
if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) {
let mut pages = doc.create_array()?;
for entry in entries {
let value = any_to_value(doc, entry)?;
pages.push(value)?;
}
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
return Ok(pages);
}
let pages = doc.create_array()?;
meta.insert("pages".to_string(), Value::Array(pages.clone()))?;
Ok(pages)
}
/// Adds a document ID to the root doc's meta.pages array.
/// Returns a binary update that can be applied to the root doc.
///
/// # Arguments
/// * `root_doc_bin` - The current root doc binary
/// * `doc_id` - The document ID to add
/// * `title` - Optional title for the document
///
/// # Returns
/// A Vec<u8> containing the y-octo update binary to add the doc
pub fn add_doc_to_root_doc(root_doc_bin: Vec<u8>, doc_id: &str, title: Option<&str>) -> Result<Vec<u8>, ParseError> {
// Handle empty or minimal root doc - create a new one
let doc = load_doc_or_new(&root_doc_bin)?;
// Capture state before modifications to encode only the delta
let state_before = doc.get_state_vector();
// Get or create the meta map
let mut meta = doc.get_or_create_map("meta")?;
let mut pages = ensure_pages_array(&doc, &mut meta)?;
// Check if doc already exists
let doc_exists = pages.iter().any(|page_val| {
page_val
.to_map()
.and_then(|page| get_string(&page, "id"))
.map(|id| id == doc_id)
.unwrap_or(false)
});
if !doc_exists {
let page_map = doc.create_map()?;
let idx = pages.len();
pages.insert(idx, Value::Map(page_map))?;
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
let page_title = title.unwrap_or(DEFAULT_DOC_TITLE);
inserted_page.insert("title".to_string(), Any::String(page_title.to_string()))?;
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0);
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
let tags = doc.create_array()?;
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
}
}
// Encode only the changes (delta) since state_before
Ok(doc.encode_state_as_update_v1(&state_before)?)
}
fn insert_page_stub(doc: &Doc, pages: &mut Array, doc_id: &str, title: Option<&str>) -> Result<(), ParseError> {
let page_map = doc.create_map()?;
let idx = pages.len();
pages.insert(idx, Value::Map(page_map))?;
if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) {
inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?;
inserted_page.insert(
"title".to_string(),
Any::String(title.unwrap_or(DEFAULT_DOC_TITLE).to_string()),
)?;
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0);
inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?;
let tags = doc.create_array()?;
inserted_page.insert("tags".to_string(), Value::Array(tags))?;
}
Ok(())
}
fn insert_simple_array(doc: &Doc, values: &[Any]) -> Result<Value, ParseError> {
let mut array = doc.create_array()?;
for value in values {
match value {
Any::Array(_) | Any::Object(_) => continue,
_ => array.push(Value::Any(value.clone()))?,
}
}
Ok(Value::Array(array))
}
fn insert_page_from_any(doc: &Doc, pages: &mut Array, page: Any) -> Result<Option<String>, ParseError> {
let Any::Object(page) = page else {
return Ok(None);
};
let Some(page_id) = page.get("id").and_then(|value| match value {
Any::String(value) => Some(value.clone()),
_ => None,
}) else {
return Ok(None);
};
let page_map = doc.create_map()?;
let idx = pages.len();
pages.insert(idx, Value::Map(page_map))?;
let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) else {
return Ok(None);
};
for key in SIMPLE_PAGE_META_KEYS {
let Some(value) = page.get(*key) else {
continue;
};
match value {
Any::Array(values) => {
inserted_page.insert((*key).to_string(), insert_simple_array(doc, values)?)?;
}
Any::Object(_) => continue,
_ => {
inserted_page.insert((*key).to_string(), Value::Any(value.clone()))?;
}
}
}
if let Some(Any::Array(tags)) = page.get("tags") {
inserted_page.insert("tags".to_string(), insert_simple_array(doc, tags)?)?;
}
Ok(Some(page_id))
}
pub fn build_public_root_doc(root_doc_bin: &[u8], doc_metas: &[(&str, Option<&str>)]) -> Result<Vec<u8>, ParseError> {
let source = load_doc_or_new(root_doc_bin)?;
let public_doc_ids = doc_metas
.iter()
.map(|(doc_id, _title)| (*doc_id).to_string())
.collect::<HashSet<_>>();
let doc = Doc::default();
let mut meta = doc.get_or_create_map("meta")?;
let mut pages = ensure_pages_array(&doc, &mut meta)?;
let mut copied = HashSet::new();
if let Ok(source_meta) = source.get_map("meta") {
let source_pages_value = source_meta.get("pages");
if let Some(source_pages) = source_pages_value.as_ref().and_then(|value| value.to_array()) {
for page_val in source_pages.iter() {
let Some(page) = page_val.to_map() else {
continue;
};
let Some(page_id) = get_string(&page, "id") else {
continue;
};
if !public_doc_ids.contains(&page_id) {
continue;
}
let page_object = Any::Object(
page
.iter()
.filter_map(|(key, value)| value.to_any().map(|any| (key.to_string(), any)))
.collect(),
);
if let Some(inserted_page_id) = insert_page_from_any(&doc, &mut pages, page_object)? {
copied.insert(inserted_page_id);
}
}
} else if let Some(Any::Array(entries)) = source_pages_value.and_then(|value| value.to_any()) {
for entry in entries {
let Any::Object(page) = entry.clone() else {
continue;
};
let Some(Any::String(page_id)) = page.get("id") else {
continue;
};
if !public_doc_ids.contains(page_id) {
continue;
}
if let Some(inserted_page_id) = insert_page_from_any(&doc, &mut pages, entry)? {
copied.insert(inserted_page_id);
}
}
}
}
for (doc_id, title) in doc_metas {
if copied.contains(*doc_id) {
continue;
}
insert_page_stub(&doc, &mut pages, doc_id, *title)?;
}
Ok(doc.encode_update_v1()?)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::doc_parser::get_doc_ids_from_binary;
#[test]
fn test_build_public_root_doc_filters_private_pages() {
let root = add_doc_to_root_doc(Vec::new(), "public-doc", Some("Public")).expect("create public entry");
let update = add_doc_to_root_doc(root.clone(), "private-doc", Some("Private")).expect("create private entry");
let mut merged = load_doc_or_new(&root).expect("load root");
merged
.apply_update_from_binary_v1(&update)
.expect("apply second update");
let merged_bin = merged.encode_update_v1().expect("encode merged");
let public_root = build_public_root_doc(
&merged_bin,
&[("public-doc", Some("Public")), ("missing-public-doc", Some("Fallback"))],
)
.expect("build public root");
let doc_ids = get_doc_ids_from_binary(public_root, false).expect("read public root doc ids");
assert_eq!(
doc_ids,
vec!["public-doc".to_string(), "missing-public-doc".to_string()]
);
}
}
packages/common/native/src/doc_parser/write/update.rs
-671
View File
@@ -1,671 +0,0 @@
//! Update YDoc module
//!
//! Provides functionality to update existing AFFiNE documents by applying
//! surgical y-octo operations based on content differences.
use std::collections::HashMap;
use super::{
super::{
block_spec::{TreeNode, count_tree_nodes, text_delta_eq},
blocksuite::{collect_child_ids, find_child_id_by_flavour},
markdown::{MAX_BLOCKS, parse_markdown_blocks},
},
builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children},
*,
};
const MAX_LCS_CELLS: usize = 2_000_000;
#[derive(Debug, Clone)]
struct StoredNode {
id: String,
spec: BlockSpec,
children: Vec<StoredNode>,
}
impl TreeNode for StoredNode {
fn children(&self) -> &[StoredNode] {
&self.children
}
}
struct DocState {
doc: Doc,
note_id: String,
blocks: Vec<StoredNode>,
}
#[derive(Debug)]
enum PatchOp {
Keep(usize, usize),
Delete(usize),
Insert(usize),
Update(usize, usize),
}
/// Updates an existing document with new markdown content.
///
/// This function performs structural diffing between the existing document
/// and the new markdown content, then applies block-level replacements
/// for changed blocks. This enables proper CRDT merging with concurrent
/// edits from other clients.
///
/// # Arguments
/// * `existing_binary` - The current document binary
/// * `new_markdown` - The new markdown content (document title is not updated)
/// * `doc_id` - The document ID
///
/// # Returns
/// A binary vector representing only the delta (changes) to apply
pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result<Vec<u8>, ParseError> {
let mut new_nodes = parse_markdown_blocks(new_markdown)?;
let state = load_doc_state(existing_binary, doc_id)?;
check_limits(&state.blocks, &new_nodes)?;
let state_before = state.doc.get_state_vector();
let mut blocks_map = state.doc.get_map("blocks")?;
let new_children = sync_nodes(&state.doc, &mut blocks_map, &state.blocks, &mut new_nodes)?;
sync_children(&state.doc, &mut blocks_map, &state.note_id, &new_children)?;
Ok(state.doc.encode_state_as_update_v1(&state_before)?)
}
fn load_doc_state(binary: &[u8], doc_id: &str) -> Result<DocState, ParseError> {
let doc = load_doc(binary, Some(doc_id))?;
let blocks_map = doc.get_map("blocks")?;
if blocks_map.is_empty() {
return Err(ParseError::ParserError("blocks map is empty".into()));
}
let block_index = build_block_index(&blocks_map);
let page_id = find_block_id_by_flavour(&block_index.block_pool, PAGE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
let page_block = block_index
.block_pool
.get(&page_id)
.ok_or_else(|| ParseError::ParserError("page block not found".into()))?;
let note_id = find_child_id_by_flavour(page_block, &block_index.block_pool, NOTE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
let note_block = block_index
.block_pool
.get(&note_id)
.ok_or_else(|| ParseError::ParserError("note block not found".into()))?;
let content_ids = collect_child_ids(note_block);
let mut blocks = Vec::new();
for block_id in content_ids {
let block = block_index
.block_pool
.get(&block_id)
.ok_or_else(|| ParseError::ParserError("content block not found".into()))?;
blocks.push(build_stored_tree(&block_id, block, &block_index.block_pool)?);
}
Ok(DocState { doc, note_id, blocks })
}
fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap<String, Map>) -> Result<StoredNode, ParseError> {
let spec = BlockSpec::from_block_map(block)?;
let child_ids = collect_child_ids(block);
if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) {
return Err(ParseError::ParserError(format!(
"unsupported children on block: {block_id}"
)));
}
let mut children = Vec::new();
for child_id in child_ids {
let child_block = pool
.get(&child_id)
.ok_or_else(|| ParseError::ParserError("child block not found".into()))?;
children.push(build_stored_tree(&child_id, child_block, pool)?);
}
Ok(StoredNode {
id: block_id.to_string(),
spec,
children,
})
}
fn sync_nodes(
doc: &Doc,
blocks_map: &mut Map,
current: &[StoredNode],
target: &mut [BlockNode],
) -> Result<Vec<String>, ParseError> {
let ops = diff_blocks(current, target);
let mut new_children = Vec::new();
let mut to_remove = Vec::new();
for op in ops {
match op {
PatchOp::Keep(old_idx, new_idx) => {
let old_node = &current[old_idx];
let new_node = &target[new_idx];
update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?;
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
new_children.push(old_node.id.clone());
}
PatchOp::Update(old_idx, new_idx) => {
let old_node = &current[old_idx];
let new_node = &target[new_idx];
update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?;
let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?;
sync_children(doc, blocks_map, &old_node.id, &child_ids)?;
new_children.push(old_node.id.clone());
}
PatchOp::Insert(new_idx) => {
let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?;
new_children.push(new_id);
}
PatchOp::Delete(old_idx) => {
let node = &current[old_idx];
if node.spec.flavour == BlockFlavour::Callout {
new_children.push(node.id.clone());
} else {
collect_tree_ids(node, &mut to_remove);
}
}
}
}
for id in to_remove {
blocks_map.remove(&id);
}
Ok(new_children)
}
fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec<PatchOp> {
let old_len = current.len();
let new_len = target.len();
if old_len == 0 {
return (0..new_len).map(PatchOp::Insert).collect();
}
if new_len == 0 {
return (0..old_len).map(PatchOp::Delete).collect();
}
let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1];
for i in 1..=old_len {
for j in 1..=new_len {
let old_spec = &current[i - 1].spec;
let new_spec = &target[j - 1].spec;
if old_spec.is_exact(new_spec) {
lcs[i][j] = lcs[i - 1][j - 1] + 1;
} else {
lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]);
}
}
}
let mut ops = Vec::new();
let mut i = old_len;
let mut j = new_len;
while i > 0 || j > 0 {
if i > 0 && j > 0 {
let old_spec = &current[i - 1].spec;
let new_spec = &target[j - 1].spec;
if old_spec.is_exact(new_spec) {
ops.push(PatchOp::Keep(i - 1, j - 1));
i -= 1;
j -= 1;
} else if old_spec.is_similar(new_spec)
&& lcs[i - 1][j - 1] >= lcs[i - 1][j]
&& lcs[i - 1][j - 1] >= lcs[i][j - 1]
{
ops.push(PatchOp::Update(i - 1, j - 1));
i -= 1;
j -= 1;
} else if lcs[i][j - 1] >= lcs[i - 1][j] {
ops.push(PatchOp::Insert(j - 1));
j -= 1;
} else {
ops.push(PatchOp::Delete(i - 1));
i -= 1;
}
} else if j > 0 {
ops.push(PatchOp::Insert(j - 1));
j -= 1;
} else {
ops.push(PatchOp::Delete(i - 1));
i -= 1;
}
}
ops.reverse();
ops
}
fn update_block_props(
doc: &Doc,
blocks_map: &mut Map,
node: &StoredNode,
target: &BlockSpec,
preserve_text: bool,
) -> Result<(), ParseError> {
let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else {
return Err(ParseError::ParserError(format!("Block {} not found", node.id)));
};
let preserve = match target.flavour {
BlockFlavour::Image
| BlockFlavour::Table
| BlockFlavour::Bookmark
| BlockFlavour::EmbedYoutube
| BlockFlavour::EmbedIframe => preserve_text,
_ => preserve_text || text_delta_eq(&node.spec.text, &target.text),
};
apply_block_spec(
doc,
&mut block,
target,
ApplyBlockOptions {
preserve_text: preserve,
clear_missing: true,
},
)?;
Ok(())
}
fn sync_children(doc: &Doc, blocks_map: &mut Map, block_id: &str, children: &[String]) -> Result<(), ParseError> {
let Some(mut block) = blocks_map.get(block_id).and_then(|v| v.to_map()) else {
return Err(ParseError::ParserError("Block not found".into()));
};
let current_children = collect_child_ids(&block);
if current_children != children {
insert_children(doc, &mut block, children)?;
}
Ok(())
}
fn collect_tree_ids(node: &StoredNode, output: &mut Vec<String>) {
output.push(node.id.clone());
for child in &node.children {
collect_tree_ids(child, output);
}
}
fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> {
let current_count = count_tree_nodes(current);
let target_count = count_tree_nodes(target);
if current_count > MAX_BLOCKS || target_count > MAX_BLOCKS {
return Err(ParseError::ParserError("block_count_too_large".into()));
}
if current_count.saturating_mul(target_count) > MAX_LCS_CELLS {
return Err(ParseError::ParserError("diff_matrix_too_large".into()));
}
Ok(())
}
#[cfg(test)]
mod tests {
use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert};
use super::{super::builder::text_ops_from_plain, *};
use crate::doc_parser::{
block_spec::BlockType, blocksuite::get_string, build_full_doc, markdown::MAX_MARKDOWN_CHARS, parse_doc_to_markdown,
};
#[test]
fn test_compute_text_diff_simple() {
let ops = text_ops_from_plain("hello world");
assert_eq!(ops.len(), 1);
match &ops[0] {
TextDeltaOp::Insert {
insert: TextInsert::Text(text),
format: None,
} => {
assert_eq!(text, "hello world");
}
_ => panic!("unexpected delta op"),
}
}
#[test]
fn test_content_block_similarity() {
let b1 = BlockSpec {
flavour: BlockFlavour::Paragraph,
block_type: Some(BlockType::H1),
text: text_ops_from_plain("Hello"),
checked: None,
language: None,
order: None,
image: None,
table: None,
bookmark: None,
embed_youtube: None,
embed_iframe: None,
};
let b2 = BlockSpec {
flavour: BlockFlavour::Paragraph,
block_type: Some(BlockType::H1),
text: text_ops_from_plain("World"),
checked: None,
language: None,
order: None,
image: None,
table: None,
bookmark: None,
embed_youtube: None,
embed_iframe: None,
};
let b3 = BlockSpec {
flavour: BlockFlavour::Paragraph,
block_type: Some(BlockType::H2),
text: text_ops_from_plain("Hello"),
checked: None,
language: None,
order: None,
image: None,
table: None,
bookmark: None,
embed_youtube: None,
embed_iframe: None,
};
assert!(b1.is_similar(&b2));
assert!(!b1.is_similar(&b3));
}
#[test]
fn test_update_ydoc_roundtrip() {
let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph.";
let doc_id = "update-test";
let initial_bin = build_full_doc("Test Document", initial_md, doc_id).expect("Should create initial doc");
let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph.";
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
assert!(!delta.is_empty(), "Delta should contain changes");
}
#[test]
fn test_update_ydoc_does_not_update_page_title() {
let initial_md = "# Original Title\n\nContent here.";
let doc_id = "title-test";
let initial_bin = build_full_doc("Original Title", initial_md, doc_id).expect("Should create initial doc");
let updated_md = "# New Title\n\nContent here.";
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
let blocks_map = doc.get_map("blocks").expect("blocks map exists");
let mut title = None;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR)
{
title = get_string(&block_map, "prop:title");
break;
}
}
assert_eq!(title.as_deref(), Some("Original Title"));
}
#[test]
fn test_update_ydoc_no_changes() {
let markdown = "# Same Title\n\nSame content.";
let doc_id = "no-change-test";
let initial_bin = build_full_doc("Same Title", markdown, doc_id).expect("Should create initial doc");
let delta = update_doc(&initial_bin, markdown, doc_id).expect("Should compute delta");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
doc
.apply_update_from_binary_v1(&delta)
.expect("Should apply delta even with no changes");
}
#[test]
fn test_update_ydoc_ignores_ai_editable_comments() {
let markdown = "Plain paragraph.";
let doc_id = "ai-comment-test";
let initial_bin = build_full_doc("Title", markdown, doc_id).expect("Should create initial doc");
let ai_markdown = parse_doc_to_markdown(initial_bin.clone(), doc_id.to_string(), true, None)
.expect("parse doc")
.markdown;
assert!(ai_markdown.contains("block_id="));
let delta = update_doc(&initial_bin, &ai_markdown, doc_id).expect("Should compute delta");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
doc.apply_update_from_binary_v1(&delta).expect("Should apply delta");
let before = parse_doc_to_markdown(initial_bin, doc_id.to_string(), false, None)
.expect("parse before")
.markdown;
let after = parse_doc_to_markdown(doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None)
.expect("parse after")
.markdown;
assert_eq!(after, before);
}
#[test]
fn test_update_ydoc_add_block() {
let initial_md = "# Add Block Test\n\nOriginal paragraph.";
let doc_id = "add-block-test";
let initial_bin = build_full_doc("Add Block Test", initial_md, doc_id).expect("Should create initial doc");
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
initial_doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added.";
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
assert!(!delta.is_empty(), "Delta should contain changes");
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
updated_doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
updated_doc
.apply_update_from_binary_v1(&delta)
.expect("Should apply delta with new block");
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
assert!(
updated_count > initial_count,
"Expected more blocks after insert, got {updated_count} vs {initial_count}"
);
}
#[test]
fn test_update_ydoc_delete_block() {
let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete.";
let doc_id = "delete-block-test";
let initial_bin = build_full_doc("Delete Block Test", initial_md, doc_id).expect("Should create initial doc");
let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
initial_doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len();
let updated_md = "# Delete Block Test\n\nFirst paragraph.";
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta");
assert!(!delta.is_empty(), "Delta should contain changes");
let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
updated_doc
.apply_update_from_binary_v1(&initial_bin)
.expect("Should apply initial");
updated_doc
.apply_update_from_binary_v1(&delta)
.expect("Should apply delta with block deletion");
let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len();
assert!(
updated_count < initial_count,
"Expected fewer blocks after deletion, got {updated_count} vs {initial_count}"
);
}
#[test]
fn test_update_ydoc_update_image_caption() {
let initial_md = "![Alt](blob://image-id)";
let doc_id = "image-update-test";
let initial_bin = build_full_doc("Image", initial_md, doc_id).expect("create doc");
let updated_md = "![New Caption](blob://image-id)";
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
let blocks_map = doc.get_map("blocks").expect("blocks map");
let mut caption = None;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image")
{
caption = get_string(&block_map, "prop:caption");
break;
}
}
assert_eq!(caption.as_deref(), Some("New Caption"));
}
#[test]
fn test_update_ydoc_update_table_cell() {
let initial_md = "| A | B |\n| --- | --- |\n| 1 | 2 |";
let doc_id = "table-update-test";
let initial_bin = build_full_doc("Table", initial_md, doc_id).expect("create doc");
let updated_md = "| A | B |\n| --- | --- |\n| 1 | 9 |";
let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta");
let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build();
doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial");
doc.apply_update_from_binary_v1(&delta).expect("apply delta");
let blocks_map = doc.get_map("blocks").expect("blocks map");
let mut found = false;
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map()
&& get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table")
{
for key in block_map.keys() {
if key.starts_with("prop:cells.")
&& key.ends_with(".text")
&& let Some(value) = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a {
Any::String(value) => Some(value),
_ => None,
})
&& value == "9"
{
found = true;
break;
}
}
}
}
assert!(found);
}
#[test]
fn test_update_ydoc_concurrent_merge_simulation() {
let base_md = "# Concurrent Test\n\nBase paragraph.";
let doc_id = "concurrent-test";
let base_bin = build_full_doc("Concurrent Test", base_md, doc_id).expect("Should create base doc");
let mut base_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
base_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
let base_count = base_doc.get_map("blocks").expect("blocks map exists").len();
let client_a_md = "# Concurrent Test\n\nModified by client A.";
let delta_a = update_doc(&base_bin, client_a_md, doc_id).expect("Delta A");
let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B.";
let delta_b = update_doc(&base_bin, client_b_md, doc_id).expect("Delta B");
let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build();
final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base");
final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A");
final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B");
let final_count = final_doc.get_map("blocks").expect("blocks map exists").len();
assert!(
final_count > base_count,
"Expected merged blocks after concurrent updates, got {final_count} vs {base_count}"
);
}
#[test]
fn test_update_ydoc_empty_binary_errors() {
let markdown = "# New Document\n\nCreated from empty binary.";
let doc_id = "empty-fallback-test";
let result = update_doc(&[], markdown, doc_id);
assert!(result.is_err());
let result = update_doc(&[0, 0], markdown, doc_id);
assert!(result.is_err());
}
#[test]
fn test_update_ydoc_markdown_too_large() {
let initial_md = "# Title\n\nContent.";
let doc_id = "size-limit-test";
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1);
let result = update_doc(&initial_bin, &markdown, doc_id);
assert!(result.is_err());
}
#[test]
fn test_update_ydoc_rejects_unsupported_markdown() {
let initial_md = "# Title\n\nContent.";
let doc_id = "unsupported-test";
let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc");
let markdown = "# Title\n\n<div>HTML</div>";
let result = update_doc(&initial_bin, markdown, doc_id);
assert!(result.is_err());
}
}
packages/common/native/src/lib.rs
-2
View File
@@ -1,5 +1,3 @@
#[cfg(feature = "ydoc-loader")]
pub mod doc_parser;
#[cfg(feature = "hashcash")]
pub mod hashcash;
#[cfg(feature = "napi")]
packages/frontend/apps/electron-renderer/src/app/native-import.ts
+15
View File
@@ -0,0 +1,15 @@
import { registerNativeImportSessionHandlers } from '@affine/core/modules/import';
import { apis } from '@affine/electron-api';
const importApis = apis?.import;
if (importApis) {
registerNativeImportSessionHandlers({
createImportSession: options =>
importApis.createImportSessionFromSource(options),
nextImportBatch: sessionId => importApis.nextImportBatch(sessionId),
cancelImportSession: sessionId => importApis.cancelImportSession(sessionId),
disposeImportSession: sessionId =>
importApis.disposeImportSession(sessionId),
});
}
packages/frontend/apps/electron-renderer/src/app/setup.ts
+1
View File
@@ -2,6 +2,7 @@ import '@affine/core/bootstrap/electron';
import '@affine/core/bootstrap/cleanup';
import '@affine/component/theme';
import './global.css';
import './native-import';
import { apis } from '@affine/electron-api';
import { bindNativeDBApis } from '@affine/nbstore/sqlite';
packages/frontend/apps/electron/package.json
+1
View File
@@ -15,6 +15,7 @@
"./main/exposed": "./src/main/exposed.ts",
"./preload/electron-api": "./src/preload/electron-api.ts",
"./preload/shared-storage": "./src/preload/shared-storage.ts",
"./shared/import": "./src/shared/import.ts",
"./main/shared-state-schema": "./src/main/shared-state-schema.ts",
"./main/updater/event": "./src/main/updater/event.ts",
"./main/windows-manager": "./src/main/windows-manager/index.ts"
packages/frontend/apps/electron/src/main/handlers.ts
+2
View File
@@ -7,6 +7,7 @@ import { byokStorageHandlers } from './byok-storage/handlers';
import { clipboardHandlers } from './clipboard';
import { configStorageHandlers } from './config-storage';
import { findInPageHandlers } from './find-in-page';
import { importHandlers } from './import';
import { getLogFilePath, logger, revealLogFile } from './logger';
import { recordingHandlers } from './recording';
import { checkSource } from './security-restrictions';
@@ -39,6 +40,7 @@ export const allHandlers = {
updater: updaterHandlers,
configStorage: configStorageHandlers,
findInPage: findInPageHandlers,
import: importHandlers,
sharedStorage: sharedStorageHandlers,
worker: workerHandlers,
recording: recordingHandlers,
packages/frontend/apps/electron/src/main/import/handlers.ts
+39
View File
@@ -0,0 +1,39 @@
import type { CreateImportSessionOptions } from '@affine/native';
import {
cancelImportSession,
createImportSession,
disposeImportSession,
nextImportBatch,
} from '@affine/native';
export const importHandlers = {
createImportSession: (
event: Electron.IpcMainInvokeEvent,
options: CreateImportSessionOptions
) => {
void event;
return createImportSession({
format: options.format,
source: options.source,
batchLimits: options.batchLimits,
});
},
nextImportBatch: (event: Electron.IpcMainInvokeEvent, sessionId: string) => {
void event;
return nextImportBatch(sessionId);
},
cancelImportSession: (
event: Electron.IpcMainInvokeEvent,
sessionId: string
) => {
void event;
return cancelImportSession(sessionId);
},
disposeImportSession: (
event: Electron.IpcMainInvokeEvent,
sessionId: string
) => {
void event;
return disposeImportSession(sessionId);
},
};
packages/frontend/apps/electron/src/main/import/index.ts
+1
View File
@@ -0,0 +1 @@
export { importHandlers } from './handlers';
packages/frontend/apps/electron/src/preload/electron-api.ts
+57 -1
View File
@@ -3,10 +3,14 @@ import type { MessagePort } from 'node:worker_threads';
import type { EventBasedChannel } from 'async-call-rpc';
import { AsyncCall } from 'async-call-rpc';
import { ipcRenderer } from 'electron';
import { ipcRenderer, webUtils } from 'electron';
import { Subject } from 'rxjs';
import { z } from 'zod';
import type {
CreateImportSessionFromSourceOptions,
NativeImportBrowserSource,
} from '../shared/import';
import {
AFFINE_API_CHANNEL_NAME,
AFFINE_EVENT_CHANNEL_NAME,
@@ -275,9 +279,61 @@ function getHelperAPIs() {
const mainAPIs = getMainAPIs();
const helperAPIs = getHelperAPIs();
type DirectoryImportFile = File & { webkitRelativePath?: string };
function filePathFromFile(file: File) {
return webUtils.getPathForFile(file);
}
function directoryPathFromFiles(files: File[]) {
const first = files.find(
(file): file is DirectoryImportFile =>
!!(file as DirectoryImportFile).webkitRelativePath
);
if (!first) return null;
const filePath = filePathFromFile(first);
if (!filePath) return null;
const relativePath = first.webkitRelativePath;
if (!relativePath) return null;
const relativeParts = relativePath.split('/');
let rootPath = filePath.replaceAll('\\', '/');
for (let i = relativeParts.length - 1; i > 0; i--) {
const part = relativeParts[i];
if (part && rootPath.endsWith(`/${part}`)) {
rootPath = rootPath.slice(0, -part.length - 1);
}
}
return rootPath || null;
}
function resolveNativeImportSource(source: NativeImportBrowserSource) {
if (source.kind === 'file') {
const path = filePathFromFile(source.file);
return path ? { kind: 'filePath', path } : null;
}
const path = directoryPathFromFiles(source.files);
return path ? { kind: 'directoryPath', path } : null;
}
export const apis = {
...mainAPIs.apis,
...helperAPIs.apis,
import: {
...mainAPIs.apis.import,
createImportSessionFromSource(
options: CreateImportSessionFromSourceOptions
) {
const source = resolveNativeImportSource(options.source);
if (!source) {
throw new Error('Native import requires a local file source');
}
return mainAPIs.apis.import.createImportSession({
format: options.format,
source,
batchLimits: options.batchLimits,
});
},
},
};
export const events = {
packages/frontend/apps/electron/src/shared/import.ts
+28
View File
@@ -0,0 +1,28 @@
export type NativeImportFormat =
| 'markdownZip'
| 'notionZip'
| 'obsidian'
| 'bearZip';
export type NativeImportBrowserSource =
| { kind: 'file'; file: File }
| { kind: 'directory'; files: File[] };
export type CreateImportSessionFromSourceOptions = {
format: NativeImportFormat;
source: NativeImportBrowserSource;
batchLimits?: {
maxDocs?: number;
maxBlobs?: number;
maxBlobBytes?: number;
};
};
export type NativeImportSessionHandlers = {
createImportSession(
options: CreateImportSessionFromSourceOptions
): Promise<string> | string;
nextImportBatch(sessionId: string): Promise<string | null> | string | null;
cancelImportSession(sessionId: string): Promise<void> | void;
disposeImportSession(sessionId: string): Promise<void> | void;
};
packages/frontend/core/src/desktop/dialogs/import/commit-service.spec.ts
+547
View File
@@ -0,0 +1,547 @@
import 'fake-indexeddb/auto';
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
import { getAFFiNEWorkspaceSchema } from '@affine/core/modules/workspace';
import type { DocSnapshot } from '@blocksuite/affine/store';
import { TestWorkspace } from '@blocksuite/affine/store/test';
import type { ImportBatch } from '@blocksuite/affine/widgets/linked-doc';
import { describe, expect, test, vi } from 'vitest';
import { ImportCommitService } from './commit-service';
function docSnapshot(id: string, title: string): DocSnapshot {
return {
type: 'page',
meta: {
id,
title,
createDate: 0,
tags: [],
},
blocks: {
type: 'block',
id: `block:${id}`,
flavour: 'affine:page',
props: {
title: {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
},
},
children: [
{
type: 'block',
id: `block:${id}:note`,
flavour: 'affine:note',
props: {},
children: [
{
type: 'block',
id: `block:${id}:paragraph`,
flavour: 'affine:paragraph',
props: {
type: 'text',
text: {
'$blocksuite:internal:text$': true,
delta: [{ insert: title }],
},
},
children: [],
},
],
},
],
},
};
}
function createFolderTree() {
const folders = new Map<string, FolderNode>();
const links: { parentId: string; docId: string }[] = [];
let nextId = 0;
class FolderNode {
readonly children: string[] = [];
constructor(readonly id: string) {
folders.set(id, this);
}
createFolder() {
const id = `folder-${++nextId}`;
this.children.push(id);
new FolderNode(id);
return id;
}
createLink(...[, docId]: ['doc', string]) {
links.push({ parentId: this.id, docId });
}
indexAt() {
return 'after';
}
}
const rootFolder = new FolderNode('root');
return {
links,
service: {
folderTree: {
rootFolder,
['folderNode$']: (id: string) => ({ value: folders.get(id) }),
},
},
};
}
function createCommitService(
collection: TestWorkspace,
options: {
organizeService?: unknown;
explorerIconService?: unknown;
tagService?: unknown;
} = {}
) {
return new ImportCommitService({
collection,
schema: getAFFiNEWorkspaceSchema(),
extensions: getStoreManager().config.init().value.get('store'),
organizeService: options.organizeService as never,
explorerIconService: options.explorerIconService as never,
tagService: options.tagService as never,
logger: {
warn: vi.fn(),
},
});
}
describe('ImportCommitService', () => {
test('commits native batch blobs, docs, folders, icons, and warnings', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const folderTree = createFolderTree();
const setIcon = vi.fn();
const service = createCommitService(collection, {
organizeService: folderTree.service,
explorerIconService: { setIcon },
});
const batch: ImportBatch = {
blobs: [
{
blobId: 'blob-1',
sourcePath: 'assets/image.png',
fileName: 'image.png',
mime: 'image/png',
bytes: new Uint8Array([1, 2, 3]),
},
],
docs: [
{
id: 'doc-1',
snapshot: docSnapshot('doc-1', 'Imported'),
meta: { title: 'Committed title', favorite: true },
},
],
folders: [
{ path: 'root-folder', name: 'Root folder' },
{
path: 'root-folder/doc-1',
name: 'Imported',
parentPath: 'root-folder',
pageId: 'doc-1',
icon: { type: 'emoji', unicode: '✅' },
},
],
warnings: [{ code: 'lossy', message: 'Dropped unsupported block' }],
done: true,
};
const result = await service.commitBatch(batch);
expect(result).toEqual({
docIds: ['doc-1'],
rootFolderId: 'folder-1',
warnings: batch.warnings,
});
await expect(collection.blobSync.get('blob-1')).resolves.toBeInstanceOf(
File
);
expect(collection.getDoc('doc-1')).not.toBeNull();
expect(collection.meta.getDocMeta('doc-1')).toMatchObject({
title: 'Committed title',
favorite: true,
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
expect(setIcon).toHaveBeenCalledWith({
where: 'doc',
id: 'doc-1',
icon: { type: 'emoji', unicode: '✅' },
});
});
test('records doc commit failures as warnings and continues remaining docs', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const service = createCommitService(collection);
const result = await service.commitBatch({
blobs: [],
docs: [
{
id: 'doc-skip',
sourcePath: 'docs/skip.md',
snapshot: { ...docSnapshot('doc-skip', 'Skip'), blocks: null },
} as never,
{
id: 'doc-ok',
sourcePath: 'docs/ok.md',
snapshot: docSnapshot('doc-ok', 'Ok'),
meta: { title: 'Ok' },
},
],
done: true,
});
expect(result.docIds).toEqual(['doc-ok']);
expect(result.warnings).toEqual([
{
code: 'skipped_doc',
sourcePath: 'docs/skip.md',
message:
'Skipped docs/skip.md: document snapshot could not be committed',
},
]);
expect(collection.getDoc('doc-ok')).not.toBeNull();
});
test('records doc meta failures as warnings without dropping committed docs', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const service = createCommitService(collection);
const originalSetDocMeta = collection.meta.setDocMeta.bind(collection.meta);
const setDocMeta = vi.spyOn(collection.meta, 'setDocMeta');
setDocMeta.mockImplementation((id, meta) => {
if (id === 'doc-meta') {
throw new Error('meta failed');
}
return originalSetDocMeta(id, meta);
});
try {
const result = await service.commitBatch({
blobs: [],
docs: [
{
id: 'doc-meta',
sourcePath: 'docs/meta.md',
snapshot: docSnapshot('doc-meta', 'Meta'),
meta: { title: 'Meta' },
},
{
id: 'doc-ok',
sourcePath: 'docs/ok.md',
snapshot: docSnapshot('doc-ok', 'Ok'),
meta: { title: 'Ok' },
},
],
done: true,
});
expect(result.docIds).toEqual(['doc-meta', 'doc-ok']);
expect(result.warnings).toEqual([
{
code: 'doc_meta_failed',
sourcePath: 'docs/meta.md',
message: 'Failed to apply metadata for docs/meta.md: meta failed',
},
]);
expect(collection.getDoc('doc-meta')).not.toBeNull();
expect(collection.getDoc('doc-ok')).not.toBeNull();
expect(collection.meta.getDocMeta('doc-ok')).toMatchObject({
title: 'Ok',
});
} finally {
setDocMeta.mockRestore();
}
});
test('commits native tag names as workspace tags', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
const tags = new Map<string, { id: string; value: string }>();
const service = createCommitService(collection, {
tagService: {
randomTagColor: () => 'red',
tagList: {
['tags$']: {
value: [],
},
createTag: (value: string) => {
const tag = { id: `tag-${tags.size + 1}`, value };
tags.set(value, tag);
return tag;
},
},
},
});
await service.commitBatch({
blobs: [],
docs: [
{
id: 'doc-1',
snapshot: docSnapshot('doc-1', 'Tagged'),
meta: { tags: ['Blue Tag'], title: 'Tagged' },
},
],
tags: [{ name: 'work/project', docIds: ['doc-1'] }],
done: true,
});
expect([...tags.values()]).toEqual([
{ id: 'tag-1', value: 'Blue Tag' },
{ id: 'tag-2', value: 'work' },
]);
expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual([
'tag-1',
'tag-2',
]);
});
test('commits batch folders, icons, and collapsed Bear root tags', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
collection.createDoc('doc-2');
const folderTree = createFolderTree();
const setIcon = vi.fn();
const tags = new Map<string, { id: string; value: string }>();
const service = createCommitService(collection, {
organizeService: folderTree.service,
explorerIconService: { setIcon },
tagService: {
randomTagColor: () => 'red',
tagList: {
['tags$']: {
value: [],
},
createTag: (value: string) => {
const tag = { id: `tag-${tags.size + 1}`, value };
tags.set(value, tag);
return tag;
},
},
},
});
const result = await service.commitBatch({
docs: [],
blobs: [],
icons: [{ docId: 'doc-1', icon: { type: 'emoji', unicode: '📘' } }],
tags: [
{ name: 'work/project', docIds: ['doc-1'] },
{ name: 'work/research', docIds: ['doc-2'] },
],
folders: [
{ path: 'Bear', name: 'Bear' },
{
path: 'Bear/Idea',
name: 'Idea',
parentPath: 'Bear',
pageId: 'doc-1',
},
],
done: true,
});
expect(result).toEqual({
docIds: [],
rootFolderId: 'folder-1',
warnings: [],
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
expect(setIcon).toHaveBeenCalledWith({
where: 'doc',
id: 'doc-1',
icon: { type: 'emoji', unicode: '📘' },
});
expect([...tags.values()]).toEqual([{ id: 'tag-1', value: 'work' }]);
expect(collection.meta.getDocMeta('doc-1')?.tags).toEqual(['tag-1']);
expect(collection.meta.getDocMeta('doc-2')?.tags).toEqual(['tag-1']);
});
test('keeps folder and doc links idempotent across repeated commits', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
const batch: ImportBatch = {
docs: [],
blobs: [],
folders: [
{ path: 'Root', name: 'Root' },
{
path: 'Root/Doc',
name: 'Doc',
parentPath: 'Root',
pageId: 'doc-1',
},
],
done: true,
};
const first = await service.commitBatch(batch);
const second = await service.commitBatch(batch);
expect(first.rootFolderId).toBe('folder-1');
expect(second.rootFolderId).toBe('folder-1');
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
});
test('resolves partial batch folder links when parent arrives later', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [
{
path: 'Root/Doc',
name: 'Doc',
parentPath: 'Root',
pageId: 'doc-1',
},
],
done: false,
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [{ path: 'Root', name: 'Root' }],
done: true,
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'doc-1' },
]);
});
test('warns and clears unresolved folders on final batch', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
const result = await service.commitBatch({
docs: [],
blobs: [],
folders: [
{
path: 'Missing/Doc',
name: 'Doc',
parentPath: 'Missing',
pageId: 'doc-1',
},
],
done: true,
});
expect(folderTree.links).toEqual([]);
expect(result.warnings).toEqual([
{
code: 'unresolved_folder',
sourcePath: 'Missing/Doc',
message:
'Skipped folder placement for Missing/Doc: parent folder was not found',
},
]);
});
test('applies icons for root-level imported pages', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('doc-1');
const folderTree = createFolderTree();
const setIcon = vi.fn();
const service = createCommitService(collection, {
organizeService: folderTree.service,
explorerIconService: { setIcon },
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [
{
path: 'Doc',
name: 'Doc',
pageId: 'doc-1',
icon: { type: 'emoji', unicode: '📌' },
},
],
done: true,
});
expect(setIcon).toHaveBeenCalledWith({
where: 'doc',
id: 'doc-1',
icon: { type: 'emoji', unicode: '📌' },
});
});
test('keeps Notion page nodes usable as child containers', async () => {
const collection = new TestWorkspace({ id: 'test' });
collection.meta.initialize();
collection.createDoc('project');
collection.createDoc('nested');
const folderTree = createFolderTree();
const service = createCommitService(collection, {
organizeService: folderTree.service,
});
await service.commitBatch({
docs: [],
blobs: [],
folders: [
{ path: 'Export', name: 'Export' },
{
path: 'Export/Project',
name: 'Project',
parentPath: 'Export',
pageId: 'project',
},
{
path: 'Export/Project/Nested',
name: 'Nested',
parentPath: 'Export/Project',
pageId: 'nested',
},
],
done: true,
});
expect(folderTree.links).toEqual([
{ parentId: 'folder-1', docId: 'project' },
{ parentId: 'folder-2', docId: 'nested' },
]);
});
});
packages/frontend/core/src/desktop/dialogs/import/commit-service.ts
+333
View File
@@ -0,0 +1,333 @@
import type { IconData } from '@affine/component';
import type { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon';
import type { OrganizeService } from '@affine/core/modules/organize';
import type { TagService } from '@affine/core/modules/tag';
import {
type ExtensionType,
type Schema,
Transformer,
type Workspace,
} from '@blocksuite/affine/store';
import type {
ImportBatch,
ImportCommitResult,
ImportFolder,
ImportIconData,
} from '@blocksuite/affine/widgets/linked-doc';
type Logger = {
warn: (message: string, ...args: unknown[]) => void;
};
type CommitServiceOptions = {
collection: Workspace;
schema: Schema;
extensions: ExtensionType[];
organizeService?: OrganizeService;
explorerIconService?: ExplorerIconService;
tagService?: TagService;
logger: Logger;
};
type EmojiIconData = Extract<IconData, { unicode: string }>;
const EMOJI_ICON_TYPE = 'emoji' as EmojiIconData['type'];
function copyToArrayBuffer(bytes: Uint8Array) {
const copy = new Uint8Array(bytes.byteLength);
copy.set(bytes);
return copy.buffer;
}
function errorMessage(error: unknown) {
return error instanceof Error
? error.message || error.name
: 'Unknown error occurred';
}
export class ImportCommitService {
private readonly folderIdByPath = new Map<string, string>();
private readonly linkedDocsByFolder = new Set<string>();
private readonly pendingFolders: ImportFolder[] = [];
constructor(private readonly options: CommitServiceOptions) {}
async commitBatch(batch: ImportBatch): Promise<ImportCommitResult> {
const warnings = [...(batch.warnings ?? [])];
for (const blob of batch.blobs) {
const bytes = new Uint8Array(blob.bytes);
await this.options.collection.blobSync.set(
blob.blobId,
new File([copyToArrayBuffer(bytes)], blob.fileName, {
type: blob.mime,
})
);
}
const transformer = new Transformer({
schema: this.options.schema,
blobCRUD: this.options.collection.blobSync,
docCRUD: {
create: (id: string) =>
this.options.collection.createDoc(id).getStore({ id }),
get: (id: string) =>
this.options.collection.getDoc(id)?.getStore({ id }) ?? null,
delete: (id: string) => this.options.collection.removeDoc(id),
},
middlewares: [],
});
const docIds: string[] = [];
const tags = new Map<string, string[]>();
for (const doc of batch.docs) {
let store: Awaited<ReturnType<Transformer['snapshotToDoc']>>;
try {
store = await transformer.snapshotToDoc(doc.snapshot);
} catch (error) {
warnings.push({
code: 'skipped_doc',
sourcePath: doc.sourcePath,
message: `Skipped ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`,
});
continue;
}
if (!store) {
warnings.push({
code: 'skipped_doc',
sourcePath: doc.sourcePath,
message: `Skipped ${doc.sourcePath ?? doc.id}: document snapshot could not be committed`,
});
continue;
}
docIds.push(store.id);
if (doc.meta && Object.keys(doc.meta).length) {
try {
const { tags: docTags, ...meta } = doc.meta;
if (Object.keys(meta).length) {
this.options.collection.meta.setDocMeta(store.id, meta);
}
for (const tag of docTags ?? []) {
const docIds = tags.get(tag) ?? [];
docIds.push(store.id);
tags.set(tag, docIds);
}
} catch (error) {
warnings.push({
code: 'doc_meta_failed',
sourcePath: doc.sourcePath,
message: `Failed to apply metadata for ${doc.sourcePath ?? doc.id}: ${errorMessage(error)}`,
});
}
}
}
for (const tag of batch.tags ?? []) {
const docIds = tags.get(tag.name) ?? [];
docIds.push(...tag.docIds);
tags.set(tag.name, docIds);
}
const rootFolderId = this.applyNativeFolders(
batch.folders ?? [],
warnings,
batch.done
);
this.applyNativeTags(tags);
this.applyNativeIcons(batch.icons);
return {
docIds,
entryId: batch.entryId,
isWorkspaceFile: batch.isWorkspaceFile,
rootFolderId,
warnings,
};
}
private applyNativeFolders(
folders: ImportFolder[],
warnings: ImportCommitResult['warnings'],
batchDone: boolean
): string | undefined {
const { organizeService } = this.options;
if (folders.length === 0) return undefined;
if (!organizeService) {
for (const folder of folders) {
if (folder.pageId && !folder.parentPath) {
this.applyIcon(folder.pageId, folder.icon);
}
}
return undefined;
}
try {
let rootFolderId: string | undefined;
this.pendingFolders.push(...folders);
let progressed = true;
while (progressed && this.pendingFolders.length) {
progressed = false;
const nextPending: ImportFolder[] = [];
const pending = this.pendingFolders.splice(0);
const childParentPaths = new Set(
pending
.map(folder => folder.parentPath)
.filter((path): path is string => !!path)
);
for (const folder of pending) {
const needsContainer =
!folder.pageId || childParentPaths.has(folder.path);
if (needsContainer && !this.folderIdByPath.has(folder.path)) {
const parent = folder.parentPath
? this.folderIdByPath.has(folder.parentPath)
? organizeService.folderTree.folderNode$(
this.folderIdByPath.get(folder.parentPath) ?? ''
).value
: null
: organizeService.folderTree.rootFolder;
if (!parent) {
nextPending.push(folder);
continue;
}
const folderId = parent.createFolder(
folder.name,
parent.indexAt('after')
);
this.folderIdByPath.set(folder.path, folderId);
rootFolderId ??= folderId;
progressed = true;
} else if (needsContainer) {
rootFolderId ??= this.folderIdByPath.get(folder.path);
}
if (folder.pageId) {
if (!this.applyFolderDocLink(folder)) {
nextPending.push(folder);
continue;
}
progressed = true;
}
}
if (progressed && nextPending.length) {
this.pendingFolders.push(...nextPending);
} else if (batchDone) {
for (const folder of nextPending) {
warnings.push({
code: 'unresolved_folder',
sourcePath: folder.path,
message: `Skipped folder placement for ${folder.path}: parent folder was not found`,
});
}
break;
} else {
this.pendingFolders.push(...nextPending);
break;
}
}
return rootFolderId;
} catch (error) {
this.options.logger.warn('Failed to commit import folders:', error);
return undefined;
}
}
private applyFolderDocLink(folder: ImportFolder): boolean {
const { organizeService } = this.options;
if (!folder.pageId) return true;
if (!folder.parentPath) {
this.applyIcon(folder.pageId, folder.icon);
return true;
}
if (!organizeService) return true;
const parentFolderId = this.folderIdByPath.get(folder.parentPath);
if (!parentFolderId) return false;
const linkKey = `${parentFolderId}:${folder.pageId}`;
if (!this.linkedDocsByFolder.has(linkKey)) {
const parent =
organizeService.folderTree.folderNode$(parentFolderId).value;
if (!parent) return false;
parent.createLink('doc', folder.pageId, parent.indexAt('after'));
this.linkedDocsByFolder.add(linkKey);
}
this.applyIcon(folder.pageId, folder.icon);
return true;
}
private applyNativeIcons(icons?: ImportBatch['icons']) {
for (const icon of icons ?? []) {
this.applyIcon(icon.docId, icon.icon);
}
}
private applyIcon(id: string, icon?: ImportIconData) {
if (!icon || !this.options.explorerIconService) return;
const iconData = toIconData(icon);
if (!iconData) return;
this.options.explorerIconService.setIcon({
where: 'doc',
id,
icon: iconData,
});
}
private applyNativeTags(tags?: Map<string, string[]>) {
const { tagService, collection } = this.options;
if (!tagService || !tags?.size) return;
try {
const existingTagMap = new Map<string, string>();
for (const tag of tagService.tagList.tags$.value) {
existingTagMap.set(tag.value$.value.toLowerCase(), tag.id);
}
const rootTagDocMap = new Map<
string,
{ displayName: string; docs: Set<string> }
>();
for (const [tagName, tagDocIds] of tags) {
const originalRoot = tagName.split('/')[0];
const key = originalRoot.toLowerCase();
let entry = rootTagDocMap.get(key);
if (!entry) {
entry = { displayName: originalRoot, docs: new Set() };
rootTagDocMap.set(key, entry);
}
for (const docId of tagDocIds) {
entry.docs.add(docId);
}
}
for (const [rootTagKey, { displayName, docs }] of rootTagDocMap) {
let tagId = existingTagMap.get(rootTagKey);
if (!tagId) {
const newTag = tagService.tagList.createTag(
displayName,
tagService.randomTagColor()
);
tagId = newTag.id;
existingTagMap.set(rootTagKey, tagId);
}
for (const docId of docs) {
const doc = collection.getDoc(docId);
const currentTags = doc?.meta?.tags ?? [];
if (!currentTags.includes(tagId)) {
collection.meta.setDocMeta(docId, {
tags: [...currentTags, tagId],
});
}
}
}
} catch (error) {
this.options.logger.warn('Failed to commit import tags:', error);
}
}
}
function toIconData(icon: ImportIconData): IconData | undefined {
if (icon.type === 'emoji') {
return {
type: EMOJI_ICON_TYPE,
unicode: icon.unicode,
};
}
return undefined;
}
packages/frontend/core/src/desktop/dialogs/import/index.tsx
+178 -476
View File
@@ -1,10 +1,4 @@
import {
Button,
IconButton,
type IconData,
IconType,
Modal,
} from '@affine/component';
import { Button, IconButton, Modal } from '@affine/component';
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks';
import { useNavigateHelper } from '@affine/core/components/hooks/use-navigate-helper';
@@ -13,9 +7,10 @@ import {
GlobalDialogService,
type WORKSPACE_DIALOG_SCHEMA,
} from '@affine/core/modules/dialogs';
import { ExplorerIconService } from '@affine/core/modules/explorer-icon/services/explorer-icon';
import { OrganizeService } from '@affine/core/modules/organize';
import { TagService } from '@affine/core/modules/tag';
import {
type ImportRunContext,
ImportService,
} from '@affine/core/modules/import';
import { UrlService } from '@affine/core/modules/url';
import {
getAFFiNEWorkspaceSchema,
@@ -28,12 +23,10 @@ import track from '@affine/track';
import { openDirectory, openFilesWith } from '@blocksuite/affine/shared/utils';
import type { Workspace } from '@blocksuite/affine/store';
import {
BearTransformer,
DocxTransformer,
HtmlTransformer,
type ImportWarning,
MarkdownTransformer,
NotionHtmlTransformer,
ObsidianTransformer,
ZipTransformer,
} from '@blocksuite/affine/widgets/linked-doc';
import {
@@ -54,6 +47,7 @@ import {
type SVGAttributes,
useCallback,
useMemo,
useRef,
useState,
} from 'react';
@@ -61,177 +55,15 @@ import * as style from './styles.css';
const logger = new DebugLogger('import');
type NotionPageIcon = {
type: 'emoji' | 'image';
content: string; // emoji unicode or image URL/data
};
type FolderHierarchy = {
name: string;
path: string;
children: Map<string, FolderHierarchy>;
pageId?: string;
parentPath?: string;
icon?: NotionPageIcon;
};
// Helper function to create folder structure using OrganizeService
function createFolderStructure(
organizeService: OrganizeService,
hierarchy: FolderHierarchy,
parentFolderId: string | null = null,
explorerIconService?: ExplorerIconService
): {
folderId: string | null;
docLinks: Array<{ folderId: string; docId: string }>;
} {
const docLinks: Array<{ folderId: string; docId: string }> = [];
const rootFolder = organizeService.folderTree.rootFolder;
function processHierarchyNode(
node: FolderHierarchy,
currentParentId: string | null
): string | null {
let currentFolderId = currentParentId;
// If this node represents a folder (has children but no pageId), create it
if (node.children.size > 0 && !node.pageId && node.name) {
const parent = currentParentId
? organizeService.folderTree.folderNode$(currentParentId).value
: rootFolder;
if (parent) {
const index = parent.indexAt('after');
currentFolderId = parent.createFolder(node.name, index);
}
}
// Process all children
for (const child of node.children.values()) {
if (child.pageId) {
// This is a document, link it to the current folder
if (currentFolderId) {
docLinks.push({ folderId: currentFolderId, docId: child.pageId });
}
// Set icon for the document if available
if (child.icon && explorerIconService) {
logger.debug('=== Setting icon for document ===');
logger.debug('Document ID:', child.pageId);
logger.debug('Icon data:', child.icon);
try {
let iconData: IconData | undefined;
if (child.icon.type === 'emoji') {
iconData = {
type: IconType.Emoji,
unicode: child.icon.content,
};
logger.debug('Created emoji icon data:', iconData);
} else if (child.icon.type === 'image') {
// For image icons, we'd need to handle blob conversion
// For now, let's skip image icons or convert them to default
// This could be enhanced later to download and convert images to blobs
logger.debug(
'Skipping image icon (not implemented):',
child.icon.content
);
iconData = undefined;
}
if (iconData) {
logger.debug('Calling explorerIconService.setIcon with:', {
where: 'doc',
id: child.pageId,
icon: iconData,
});
explorerIconService.setIcon({
where: 'doc',
id: child.pageId,
icon: iconData,
});
logger.debug('Icon set successfully for document:', child.pageId);
} else {
logger.debug('No valid icon data to set');
}
} catch (error) {
logger.error(
'Error setting icon for document:',
child.pageId,
error
);
logger.warn(
'Failed to set icon for document:',
child.pageId,
error
);
}
} else {
if (!child.icon) {
logger.debug('No icon found for document:', child.pageId);
}
if (!explorerIconService) {
logger.debug(
'ExplorerIconService not available for document:',
child.pageId
);
}
}
} else if (child.children.size > 0) {
// This is a subfolder, process it recursively
processHierarchyNode(child, currentFolderId);
}
}
return currentFolderId;
}
const rootFolderId = processHierarchyNode(hierarchy, parentFolderId);
return { folderId: rootFolderId, docLinks };
}
/**
* Creates the folder tree described by {@link folderHierarchy} via
* {@link OrganizeService} and links every document into its folder.
* Returns the root folder ID on success, or `undefined` if the
* hierarchy is empty or an error occurs.
*
* When {@link explorerIconService} is provided, document icons from the
* hierarchy (e.g. Notion page emojis) are applied. Callers that do not
* need icon support can omit it safely.
*/
function applyFolderHierarchy(
organizeService: OrganizeService,
folderHierarchy: FolderHierarchy,
explorerIconService?: ExplorerIconService
): string | undefined {
if (folderHierarchy.children.size === 0) return undefined;
try {
const { folderId, docLinks } = createFolderStructure(
organizeService,
folderHierarchy,
null,
explorerIconService
);
for (const { folderId, docId } of docLinks) {
const folder = organizeService.folderTree.folderNode$(folderId).value;
if (folder) {
const index = folder.indexAt('after');
folder.createLink('doc', docId, index);
}
}
return folderId || undefined;
} catch (error) {
logger.warn('Failed to create folder structure:', error);
return undefined;
}
function shouldSnapshotPickedFiles(type: ImportType, acceptType: AcceptType) {
if (acceptType === 'Directory' || acceptType === 'Skip') return false;
return !['markdownZip', 'notion', 'bear'].includes(type);
}
type ImportType =
| 'markdown'
| 'markdownZip'
| 'notion'
| 'notionMarkdown'
| 'obsidian'
| 'bear'
| 'snapshot'
@@ -240,28 +72,71 @@ type ImportType =
| 'dotaffinefile';
type AcceptType = 'Markdown' | 'Zip' | 'Html' | 'Docx' | 'Directory' | 'Skip'; // Skip is used for dotaffinefile
type Status = 'idle' | 'importing' | 'success' | 'error';
type ImportErrorState = {
code: string;
message: string;
sourcePath?: string;
};
type ImportResult = {
docIds: string[];
entryId?: string;
isWorkspaceFile?: boolean;
rootFolderId?: string;
importedWorkspace?: WorkspaceMetadata;
warnings?: ImportWarning[];
};
type ImportedWorkspacePayload = {
workspace: WorkspaceMetadata;
};
type ImportFunctionArgs = {
docCollection: Workspace;
files: File[];
importAffineFile: () => Promise<WorkspaceMetadata | undefined>;
importService?: ImportService;
context: ImportRunContext;
};
function toImportErrorState(error: unknown): ImportErrorState {
const sourcePath =
typeof error === 'object' &&
error !== null &&
'sourcePath' in error &&
typeof error.sourcePath === 'string'
? error.sourcePath
: undefined;
if (error instanceof DOMException && error.name === 'AbortError') {
return {
code: 'cancelled',
message: 'Import cancelled',
sourcePath,
};
}
if (error instanceof Error) {
return {
code: error.name || 'import-error',
message: error.message || 'Unknown error occurred',
sourcePath,
};
}
return {
code: 'unknown',
message: 'Unknown error occurred',
sourcePath,
};
}
function requireImportService(importService?: ImportService) {
if (!importService) {
throw new Error('Import service is unavailable');
}
return importService;
}
type ImportConfig = {
fileOptions: { acceptType: AcceptType; multiple: boolean };
importFunction: (
docCollection: Workspace,
files: File[],
handleImportAffineFile: () => Promise<WorkspaceMetadata | undefined>,
organizeService?: OrganizeService,
explorerIconService?: ExplorerIconService,
tagService?: TagService
) => Promise<ImportResult>;
importFunction: (args: ImportFunctionArgs) => Promise<ImportResult>;
};
const importOptions = [
@@ -319,17 +194,6 @@ const importOptions = [
testId: 'editor-option-menu-import-notion',
type: 'notion' as ImportType,
},
{
key: 'notionMarkdown',
label: 'com.affine.import.notion-markdown',
prefixIcon: <NotionIcon color={cssVar('black')} width={20} height={20} />,
suffixIcon: (
<HelpIcon color={cssVarV2('icon/primary')} width={20} height={20} />
),
suffixTooltip: 'com.affine.import.notion-markdown.tooltip',
testId: 'editor-option-menu-import-notion-markdown',
type: 'notionMarkdown' as ImportType,
},
{
key: 'obsidian',
label: 'com.affine.import.obsidian',
@@ -400,13 +264,7 @@ const importOptions = [
const importConfigs: Record<ImportType, ImportConfig> = {
markdown: {
fileOptions: { acceptType: 'Markdown', multiple: true },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
importFunction: async ({ docCollection, files }) => {
const docIds: string[] = [];
for (const file of files) {
const text = await file.text();
@@ -427,45 +285,20 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
markdownZip: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
_explorerIconService
) => {
importFunction: async ({ files, importService, context }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for markdownZip import');
}
const { docIds, folderHierarchy } =
await MarkdownTransformer.importMarkdownZip({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(organizeService, folderHierarchy)
: undefined;
return {
docIds,
rootFolderId,
};
return requireImportService(importService).importMarkdownZip(
file,
context
);
},
},
html: {
fileOptions: { acceptType: 'Html', multiple: true },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
importFunction: async ({ docCollection, files }) => {
const docIds: string[] = [];
for (const file of files) {
const text = await file.text();
@@ -486,220 +319,39 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
notion: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
explorerIconService
) => {
importFunction: async ({ files, importService, context }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for notion import');
}
const { entryId, pageIds, isWorkspaceFile, folderHierarchy } =
await NotionHtmlTransformer.importNotionZip({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(
organizeService,
folderHierarchy,
explorerIconService
)
: undefined;
return {
docIds: pageIds,
entryId,
isWorkspaceFile,
rootFolderId,
};
},
},
notionMarkdown: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
explorerIconService
) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for notionMarkdown import');
}
const { docIds, folderHierarchy } =
await MarkdownTransformer.importNotionMarkdownZip({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(
organizeService,
folderHierarchy,
explorerIconService
)
: undefined;
return {
docIds,
rootFolderId,
};
return requireImportService(importService).importNotionZip(file, context);
},
},
obsidian: {
fileOptions: { acceptType: 'Directory', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
explorerIconService
) => {
const { docIds, docEmojis } =
await ObsidianTransformer.importObsidianVault({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
importedFiles: files,
extensions: getStoreManager().config.init().value.get('store'),
});
if (explorerIconService) {
for (const [id, emoji] of docEmojis.entries()) {
explorerIconService.setIcon({
where: 'doc',
id,
icon: { type: IconType.Emoji, unicode: emoji },
});
}
}
return { docIds };
importFunction: async ({ files, importService, context }) => {
return requireImportService(importService).importObsidianVault(
files,
context
);
},
},
bear: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
organizeService,
_explorerIconService,
tagService
) => {
importFunction: async ({ files, importService, context }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single .bear2bk file for Bear import');
}
let docIds: string[];
let tags: Map<string, string[]>;
let folderHierarchy: FolderHierarchy;
try {
const result = await BearTransformer.importBearBackup({
collection: docCollection,
schema: getAFFiNEWorkspaceSchema(),
imported: file,
extensions: getStoreManager().config.init().value.get('store'),
});
docIds = result.docIds;
tags = result.tags;
folderHierarchy = result.folderHierarchy;
} catch (err) {
logger.error('Bear import failed:', err);
throw err instanceof Error
? err
: new Error(String(err) || 'Bear import failed');
}
// Create AFFiNE tags from Bear tags
if (tagService && tags.size > 0) {
try {
// Get existing tags for deduplication
const existingTags = tagService.tagList.tags$.value;
const existingTagMap = new Map<string, string>(); // lowercase name → tag id
for (const tag of existingTags) {
const name = tag.value$.value.toLowerCase();
existingTagMap.set(name, tag.id);
}
// Consolidate tags by root segment (e.g., "privat/bike" → "privat").
// Keyed by lowercase root for case-insensitive dedup, but the
// original capitalization of the first occurrence is preserved
// so new AFFiNE tags are created with the user's casing.
const rootTagDocMap = new Map<
string,
{ displayName: string; docs: Set<string> }
>();
for (const [tagName, tagDocIds] of tags) {
const originalRoot = tagName.split('/')[0];
const key = originalRoot.toLowerCase();
let entry = rootTagDocMap.get(key);
if (!entry) {
entry = { displayName: originalRoot, docs: new Set<string>() };
rootTagDocMap.set(key, entry);
}
for (const docId of tagDocIds) {
entry.docs.add(docId);
}
}
for (const [
rootTagKey,
{ displayName, docs: docIdSet },
] of rootTagDocMap) {
// Check if tag already exists (case-insensitive)
let tagId = existingTagMap.get(rootTagKey);
if (!tagId) {
const newTag = tagService.tagList.createTag(
displayName,
tagService.randomTagColor()
);
tagId = newTag.id;
existingTagMap.set(rootTagKey, tagId);
}
// Assign tag to each doc
for (const docId of docIdSet) {
const doc = docCollection.getDoc(docId);
const currentTags = doc?.meta?.tags ?? [];
if (!currentTags.includes(tagId)) {
docCollection.meta.setDocMeta(docId, {
tags: [...currentTags, tagId],
});
}
}
}
} catch (error) {
logger.warn('Failed to create Bear tags:', error);
}
}
const rootFolderId =
folderHierarchy && organizeService
? applyFolderHierarchy(organizeService, folderHierarchy)
: undefined;
return {
docIds,
rootFolderId,
};
return requireImportService(importService).importBearBackup(
file,
context
);
},
},
docx: {
fileOptions: { acceptType: 'Docx', multiple: false },
importFunction: async (docCollection, file) => {
const files = Array.isArray(file) ? file : [file];
importFunction: async ({ docCollection, files }) => {
const docIds: string[] = [];
for (const file of files) {
const docId = await DocxTransformer.importDocx({
@@ -715,13 +367,7 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
snapshot: {
fileOptions: { acceptType: 'Zip', multiple: false },
importFunction: async (
docCollection,
files,
_handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
importFunction: async ({ docCollection, files }) => {
const file = files.length === 1 ? files[0] : null;
if (!file) {
throw new Error('Expected a single zip file for snapshot import');
@@ -743,14 +389,8 @@ const importConfigs: Record<ImportType, ImportConfig> = {
},
dotaffinefile: {
fileOptions: { acceptType: 'Skip', multiple: false },
importFunction: async (
_,
__,
handleImportAffineFile,
_organizeService,
_explorerIconService
) => {
const workspace = await handleImportAffineFile();
importFunction: async ({ importAffineFile }) => {
const workspace = await importAffineFile();
return {
docIds: [],
entryId: undefined,
@@ -843,8 +483,18 @@ const ImportOptions = ({
);
};
const ImportingStatus = () => {
const ImportingStatus = ({
progress,
onCancel,
}: {
progress: { completed: number; total: number } | null;
onCancel: () => void;
}) => {
const t = useI18n();
const progressLabel =
progress && progress.total > 0
? `${progress.completed}/${progress.total}`
: null;
return (
<>
<div className={style.importModalTitle}>
@@ -853,11 +503,25 @@ const ImportingStatus = () => {
<p className={style.importStatusContent}>
{t['com.affine.import.status.importing.message']()}
</p>
{progressLabel ? (
<div className={style.importProgress}>{progressLabel}</div>
) : null}
<div className={style.importModalButtonContainer}>
<Button onClick={onCancel} variant="secondary">
{t['Cancel']()}
</Button>
</div>
</>
);
};
const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => {
const SuccessStatus = ({
warnings,
onComplete,
}: {
warnings: string[];
onComplete: () => void;
}) => {
const t = useI18n();
return (
<>
@@ -876,6 +540,13 @@ const SuccessStatus = ({ onComplete }: { onComplete: () => void }) => {
</a>
.
</p>
{warnings.length ? (
<div className={style.importWarnings}>
{warnings.map((warning, index) => (
<div key={`${warning}-${index}`}>{warning}</div>
))}
</div>
) : null}
<div className={style.importModalButtonContainer}>
<Button onClick={onComplete} variant="primary">
{t['Complete']()}
@@ -889,7 +560,7 @@ const ErrorStatus = ({
error,
onRetry,
}: {
error: string | null;
error: ImportErrorState | null;
onRetry: () => void;
}) => {
const t = useI18n();
@@ -900,8 +571,11 @@ const ErrorStatus = ({
{t['com.affine.import.status.failed.title']()}
</div>
<p className={style.importStatusContent}>
{error || 'Unknown error occurred'}
{error?.message || 'Unknown error occurred'}
</p>
{error?.sourcePath ? (
<div className={style.importErrorDetail}>{error.sourcePath}</div>
) : null}
<div className={style.importModalButtonContainer}>
<Button
onClick={() => {
@@ -924,13 +598,16 @@ export const ImportDialog = ({
}: DialogComponentProps<WORKSPACE_DIALOG_SCHEMA['import']>) => {
const t = useI18n();
const [status, setStatus] = useState<Status>('idle');
const [importError, setImportError] = useState<string | null>(null);
const [importError, setImportError] = useState<ImportErrorState | null>(null);
const [importResult, setImportResult] = useState<ImportResult | null>(null);
const [importProgress, setImportProgress] = useState<{
completed: number;
total: number;
} | null>(null);
const importAbortControllerRef = useRef<AbortController | null>(null);
const workspace = useService(WorkspaceService).workspace;
const docCollection = workspace.docCollection;
const organizeService = useService(OrganizeService);
const explorerIconService = useService(ExplorerIconService);
const tagService = useService(TagService);
const importService = useService(ImportService);
const globalDialogService = useService(GlobalDialogService);
@@ -984,6 +661,7 @@ export const ImportDialog = ({
const handleImport = useAsyncCallback(
async (type: ImportType) => {
setImportError(null);
setImportProgress(null);
try {
const importConfig = importConfigs[type];
const { acceptType, multiple } = importConfig.fileOptions;
@@ -992,8 +670,13 @@ export const ImportDialog = ({
acceptType === 'Skip'
? []
: acceptType === 'Directory'
? await openDirectory()
: await openFilesWith(acceptType, multiple);
? await openDirectory({
fileSystemAccess: false,
})
: await openFilesWith(acceptType, multiple, {
fileSystemAccess: false,
snapshot: shouldSnapshotPickedFiles(type, acceptType),
});
if (!files || (files.length === 0 && acceptType !== 'Skip')) {
throw new Error(
@@ -1009,20 +692,28 @@ export const ImportDialog = ({
});
}
const abortController = new AbortController();
importAbortControllerRef.current = abortController;
const {
docIds,
entryId,
isWorkspaceFile,
rootFolderId,
importedWorkspace,
} = await importConfig.importFunction(
warnings,
} = await importConfig.importFunction({
docCollection,
files,
handleImportAffineFile,
organizeService,
explorerIconService,
tagService
);
importAffineFile: handleImportAffineFile,
importService,
context: {
signal: abortController.signal,
onProgress: progress => {
setImportProgress(progress);
},
},
});
importAbortControllerRef.current = null;
setImportResult({
docIds,
@@ -1030,6 +721,7 @@ export const ImportDialog = ({
isWorkspaceFile,
rootFolderId,
importedWorkspace,
warnings,
});
setStatus('success');
track.$.importModal.$.import({
@@ -1043,26 +735,19 @@ export const ImportDialog = ({
control: 'import',
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : 'Unknown error occurred';
setImportError(errorMessage);
importAbortControllerRef.current = null;
const structuredError = toImportErrorState(error);
setImportError(structuredError);
setStatus('error');
track.$.importModal.$.import({
type,
status: 'failed',
error: errorMessage || undefined,
error: structuredError.message || undefined,
});
logger.error('Failed to import', error);
}
},
[
docCollection,
explorerIconService,
handleImportAffineFile,
organizeService,
tagService,
t,
]
[docCollection, handleImportAffineFile, importService, t]
);
const finishImport = useCallback(() => {
@@ -1073,8 +758,11 @@ export const ImportDialog = ({
close();
return;
}
const { importedWorkspace: _workspace, ...result } = importResult;
close(result);
close({
docIds: importResult.docIds,
entryId: importResult.entryId,
isWorkspaceFile: importResult.isWorkspaceFile,
});
}, [close, handleCreatedWorkspace, importResult]);
const handleComplete = useCallback(() => {
@@ -1082,13 +770,27 @@ export const ImportDialog = ({
}, [finishImport]);
const handleRetry = () => {
setImportProgress(null);
setStatus('idle');
};
const handleCancel = useCallback(() => {
importAbortControllerRef.current?.abort();
}, []);
const statusComponents = {
idle: <ImportOptions onImport={handleImport} />,
importing: <ImportingStatus />,
success: <SuccessStatus onComplete={handleComplete} />,
importing: (
<ImportingStatus progress={importProgress} onCancel={handleCancel} />
),
success: (
<SuccessStatus
warnings={(importResult?.warnings ?? []).map(warning =>
typeof warning === 'string' ? warning : warning.message
)}
onComplete={handleComplete}
/>
),
error: <ErrorStatus error={importError} onRetry={handleRetry} />,
};
packages/frontend/core/src/desktop/dialogs/import/native-backend.ts
+111
View File
@@ -0,0 +1,111 @@
import {
getNativeImportSessionHandlers,
type NativeImportSessionHandlers,
} from '@affine/core/modules/import/runtime-config';
import type {
NativeImportBrowserSource,
NativeImportFormat,
} from '@affine/electron-api';
import type {
ImportBatch,
ImportCommitResult,
} from '@blocksuite/affine/widgets/linked-doc';
import type { ImportCommitService } from './commit-service';
const NATIVE_IMPORT_BATCH_LIMITS = {
maxDocs: 20,
maxBlobs: 20,
maxBlobBytes: 10 * 1024 * 1024,
};
export async function commitNativeImport(
format: NativeImportFormat,
source: File | File[],
commitService: ImportCommitService,
options: {
signal?: AbortSignal;
onProgress?: (progress: { completed: number; total: number }) => void;
} = {}
): Promise<ImportCommitResult> {
const native = await getNativeImporter();
if (options.signal?.aborted) {
throw new DOMException('Import cancelled', 'AbortError');
}
const sessionId = await native.createImportSession({
format,
source: getNativeImportSource(source),
batchLimits: NATIVE_IMPORT_BATCH_LIMITS,
});
const docIds: string[] = [];
const warnings: ImportCommitResult['warnings'] = [];
let entryId: string | undefined;
let isWorkspaceFile = false;
let rootFolderId: string | undefined;
let cancelled = false;
try {
for (;;) {
if (options.signal?.aborted) {
cancelled = true;
try {
await native.cancelImportSession(sessionId);
} catch {
// Preserve the AbortError reported to callers.
}
throw new DOMException('Import cancelled', 'AbortError');
}
const payload = await native.nextImportBatch(sessionId);
if (!payload) break;
const batch = JSON.parse(payload) as ImportBatch;
options.onProgress?.(batch.progress ?? { completed: 0, total: 0 });
const result = await commitService.commitBatch(batch);
docIds.push(...result.docIds);
warnings.push(...result.warnings);
entryId ??= batch.entryId;
isWorkspaceFile ||= !!batch.isWorkspaceFile;
rootFolderId ??= result.rootFolderId;
}
} catch (error) {
if (!cancelled) {
try {
await native.cancelImportSession(sessionId);
} catch {
// Preserve the original import error.
}
}
throw error;
} finally {
await native.disposeImportSession(sessionId);
}
if (!docIds.length && !isWorkspaceFile) {
throw new Error('No importable documents were found in the selected file.');
}
return {
docIds,
entryId,
isWorkspaceFile,
rootFolderId,
warnings,
};
}
async function getNativeImporter(): Promise<NativeImportSessionHandlers> {
const registered = getNativeImportSessionHandlers();
if (registered) {
return registered;
}
throw new Error('Native import session handlers are not registered');
}
function getNativeImportSource(
source: File | File[]
): NativeImportBrowserSource {
if (Array.isArray(source)) {
return { kind: 'directory', files: source };
}
return { kind: 'file', file: source };
}
packages/frontend/core/src/desktop/dialogs/import/styles.css.ts
+24
View File
@@ -57,6 +57,30 @@ export const importStatusContent = style({
color: cssVar('textPrimaryColor'),
});
export const importProgress = style({
width: '100%',
fontSize: cssVar('fontSm'),
lineHeight: cssVar('lineHeight'),
color: cssVar('textSecondaryColor'),
});
export const importWarnings = style({
width: '100%',
maxHeight: '120px',
overflow: 'auto',
fontSize: cssVar('fontSm'),
lineHeight: cssVar('lineHeight'),
color: cssVar('textSecondaryColor'),
});
export const importErrorDetail = style({
width: '100%',
fontSize: cssVar('fontXs'),
lineHeight: cssVar('lineHeight'),
color: cssVar('textSecondaryColor'),
overflowWrap: 'anywhere',
});
export const importModalButtonContainer = style({
width: '100%',
display: 'flex',
packages/frontend/core/src/desktop/dialogs/import/web-limits.spec.ts
+106
View File
@@ -0,0 +1,106 @@
import { describe, expect, test } from 'vitest';
import {
preflightWebFilesImport,
preflightWebZipImport,
WebImportLimitError,
} from './web-limits';
function zipDirectoryFile(names: string[]) {
const chunks = names.map(name => {
const nameBytes = new TextEncoder().encode(name);
const header = new Uint8Array(46 + nameBytes.length);
writeU32(header, 0, 0x02014b50);
writeU32(header, 20, 1);
writeU32(header, 24, 1);
writeU16(header, 28, nameBytes.length);
header.set(nameBytes, 46);
return header;
});
return new File(chunks, 'import.zip', { type: 'application/zip' });
}
function writeU16(bytes: Uint8Array, offset: number, value: number) {
bytes[offset] = value & 0xff;
bytes[offset + 1] = (value >> 8) & 0xff;
}
function writeU32(bytes: Uint8Array, offset: number, value: number) {
bytes[offset] = value & 0xff;
bytes[offset + 1] = (value >> 8) & 0xff;
bytes[offset + 2] = (value >> 16) & 0xff;
bytes[offset + 3] = (value >> 24) & 0xff;
}
describe('preflightWebZipImport', () => {
test('rejects nested zip before JS importer parsing', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['entry.md', 'nested.zip']))
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('rejects document count over web limit', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['a.md', 'b.md']), {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('counts html pages as web import documents', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['a.html', 'b.htm']), {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('allows small zip directory', async () => {
await expect(
preflightWebZipImport(zipDirectoryFile(['entry.md']), {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).resolves.toBeUndefined();
});
});
describe('preflightWebFilesImport', () => {
test('rejects directory import over web document limit', async () => {
await expect(
preflightWebFilesImport(
[new File(['a'], 'a.md'), new File(['b'], 'b.md')],
{
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
}
)
).rejects.toBeInstanceOf(WebImportLimitError);
});
test('allows small directory import', async () => {
await expect(
preflightWebFilesImport([new File(['a'], 'a.md')], {
maxTotalBytes: 1024,
maxEntryBytes: 1024,
maxEntryCount: 10,
maxNestedZipDepth: 0,
maxDocumentCount: 1,
})
).resolves.toBeUndefined();
});
});
packages/frontend/core/src/desktop/dialogs/import/web-limits.ts
+155
View File
@@ -0,0 +1,155 @@
export type WebImportLimits = {
maxTotalBytes: number;
maxEntryBytes: number;
maxEntryCount: number;
maxNestedZipDepth: number;
maxDocumentCount: number;
};
export const webImportLimits: WebImportLimits = {
maxTotalBytes: 32 * 1024 * 1024,
maxEntryBytes: 8 * 1024 * 1024,
maxEntryCount: 1000,
maxNestedZipDepth: 0,
maxDocumentCount: 250,
};
export class WebImportLimitError extends Error {
constructor(message: string) {
super(message);
this.name = 'WebImportLimitError';
}
}
export async function preflightWebZipImport(
file: File,
limits = webImportLimits
) {
if (file.size > limits.maxTotalBytes) {
throw new WebImportLimitError(
'This import is too large for the web app. Please import it in the desktop client.'
);
}
const entries = readZipDirectory(new Uint8Array(await file.arrayBuffer()));
if (entries.length > limits.maxEntryCount) {
throw new WebImportLimitError(
'This import has too many files for the web app. Please import it in the desktop client.'
);
}
const documentCount = entries.filter(entry =>
isWebImportDocument(entry.name)
).length;
if (documentCount > limits.maxDocumentCount) {
throw new WebImportLimitError(
'This import has too many documents for the web app. Please import it in the desktop client.'
);
}
for (const entry of entries) {
if (entry.uncompressedSize > limits.maxEntryBytes) {
throw new WebImportLimitError(
'This import contains a file that is too large for the web app. Please import it in the desktop client.'
);
}
if (
limits.maxNestedZipDepth === 0 &&
entry.name.toLowerCase().endsWith('.zip')
) {
throw new WebImportLimitError(
'This import contains a nested zip. Please import it in the desktop client.'
);
}
}
}
export async function preflightWebFilesImport(
files: File[],
limits = webImportLimits
) {
const totalBytes = files.reduce((total, file) => total + file.size, 0);
if (totalBytes > limits.maxTotalBytes) {
throw new WebImportLimitError(
'This import is too large for the web app. Please import it in the desktop client.'
);
}
if (files.length > limits.maxEntryCount) {
throw new WebImportLimitError(
'This import has too many files for the web app. Please import it in the desktop client.'
);
}
let documentCount = 0;
for (const file of files) {
if (file.size > limits.maxEntryBytes) {
throw new WebImportLimitError(
'This import contains a file that is too large for the web app. Please import it in the desktop client.'
);
}
const path = file.webkitRelativePath || file.name;
if (path.toLowerCase().endsWith('.zip')) {
throw new WebImportLimitError(
'This import contains a nested zip. Please import it in the desktop client.'
);
}
if (isWebImportDocument(path)) {
documentCount += 1;
}
}
if (documentCount > limits.maxDocumentCount) {
throw new WebImportLimitError(
'This import has too many documents for the web app. Please import it in the desktop client.'
);
}
}
function isWebImportDocument(path: string) {
const lower = path.toLowerCase();
return (
lower.endsWith('.md') || lower.endsWith('.html') || lower.endsWith('.htm')
);
}
type ZipEntry = {
name: string;
uncompressedSize: number;
};
function readZipDirectory(bytes: Uint8Array): ZipEntry[] {
const entries: ZipEntry[] = [];
for (let offset = 0; offset + 46 <= bytes.length; offset++) {
if (readU32(bytes, offset) !== 0x02014b50) continue;
const compressedSize = readU32(bytes, offset + 20);
const uncompressedSize = readU32(bytes, offset + 24);
const fileNameLength = readU16(bytes, offset + 28);
const extraLength = readU16(bytes, offset + 30);
const commentLength = readU16(bytes, offset + 32);
const nameStart = offset + 46;
const nameEnd = nameStart + fileNameLength;
if (nameEnd > bytes.length) break;
entries.push({
name: new TextDecoder().decode(bytes.subarray(nameStart, nameEnd)),
uncompressedSize: uncompressedSize || compressedSize,
});
offset = nameEnd + extraLength + commentLength - 1;
}
return entries;
}
function readU16(bytes: Uint8Array, offset: number) {
return bytes[offset] | (bytes[offset + 1] << 8);
}
function readU32(bytes: Uint8Array, offset: number) {
return (
(bytes[offset] |
(bytes[offset + 1] << 8) |
(bytes[offset + 2] << 16) |
(bytes[offset + 3] << 24)) >>>
0
);
}
packages/frontend/core/src/modules/import/index.ts
+30
View File
@@ -0,0 +1,30 @@
import type { Framework } from '@toeverything/infra';
import { ExplorerIconService } from '../explorer-icon/services/explorer-icon';
import { OrganizeService } from '../organize';
import { TagService } from '../tag';
import { WorkspaceScope, WorkspaceService } from '../workspace';
import { ImportService } from './services/service';
export { ImportService };
export type { NativeImportSessionHandlers } from './runtime-config';
export {
getNativeImportSessionHandlers,
registerNativeImportSessionHandlers,
} from './runtime-config';
export type { ImportRunContext } from './services/service';
export type {
NativeImportBrowserSource,
NativeImportFormat,
} from '@affine/electron-api';
export function configureImportModule(framework: Framework) {
framework
.scope(WorkspaceScope)
.service(ImportService, [
WorkspaceService,
OrganizeService,
ExplorerIconService,
TagService,
]);
}
packages/frontend/core/src/modules/import/runtime-config.ts
+19
View File
@@ -0,0 +1,19 @@
import type { NativeImportSessionHandlers } from '@affine/electron-api';
export type {
NativeImportBrowserSource,
NativeImportFormat,
NativeImportSessionHandlers,
} from '@affine/electron-api';
let nativeImportSessionHandlers: NativeImportSessionHandlers | null = null;
export function registerNativeImportSessionHandlers(
handlers: NativeImportSessionHandlers | null
) {
nativeImportSessionHandlers = handlers;
}
export function getNativeImportSessionHandlers() {
return nativeImportSessionHandlers;
}
packages/frontend/core/src/modules/import/services/service.ts
+190
View File
@@ -0,0 +1,190 @@
import { getStoreManager } from '@affine/core/blocksuite/manager/store';
import { ImportCommitService } from '@affine/core/desktop/dialogs/import/commit-service';
import { commitNativeImport } from '@affine/core/desktop/dialogs/import/native-backend';
import {
preflightWebFilesImport,
preflightWebZipImport,
} from '@affine/core/desktop/dialogs/import/web-limits';
import { DebugLogger } from '@affine/debug';
import { snapshotFile } from '@blocksuite/affine/shared/utils';
import {
BearTransformer,
type ImportWarning,
MarkdownTransformer,
NotionHtmlTransformer,
ObsidianTransformer,
Unzip,
} from '@blocksuite/affine/widgets/linked-doc';
import { Service } from '@toeverything/infra';
import type { ExplorerIconService } from '../../explorer-icon/services/explorer-icon';
import type { OrganizeService } from '../../organize';
import type { TagService } from '../../tag';
import type { WorkspaceService } from '../../workspace';
import { getAFFiNEWorkspaceSchema } from '../../workspace';
const logger = new DebugLogger('import');
export type ImportRunContext = {
signal?: AbortSignal;
onProgress?: (progress: { completed: number; total: number }) => void;
};
export class ImportService extends Service {
constructor(
private readonly workspaceService: WorkspaceService,
private readonly organizeService: OrganizeService,
private readonly explorerIconService: ExplorerIconService,
private readonly tagService: TagService
) {
super();
}
async importMarkdownZip(file: File, context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({ organize: true });
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('markdownZip', file, commitService, context);
}
await preflightWebZipImport(file);
const snapshot = await snapshotFile(file);
const { batch } = await MarkdownTransformer.planMarkdownZip({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
async importNotionZip(file: File, context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({
organize: true,
explorerIcon: true,
});
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('notionZip', file, commitService, context);
}
await preflightWebZipImport(file);
const snapshot = await snapshotFile(file);
const format = await detectNotionZipFormat(snapshot);
if (format === 'markdown') {
const { batch } = await MarkdownTransformer.planNotionMarkdownZip({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
const { batch } = await NotionHtmlTransformer.planNotionHtmlZip({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
async importObsidianVault(files: File[], context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({
explorerIcon: true,
});
if (!BUILD_CONFIG.isElectron) {
await preflightWebFilesImport(files);
}
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('obsidian', files, commitService, context);
}
const { files: snapshots, warnings } = await snapshotReadableFiles(files);
if (!snapshots.length) {
throw new Error('No readable files were found in the selected folder.');
}
const { batch } = await ObsidianTransformer.planObsidianVault({
collection,
schema: getAFFiNEWorkspaceSchema(),
importedFiles: snapshots,
extensions: getStoreManager().config.init().value.get('store'),
});
batch.warnings = [...(batch.warnings ?? []), ...warnings];
return commitService.commitBatch(batch);
}
async importBearBackup(file: File, context?: ImportRunContext) {
const collection = this.workspaceService.workspace.docCollection;
const commitService = this.createCommitService({
organize: true,
tag: true,
});
if (BUILD_CONFIG.isElectron) {
return commitNativeImport('bearZip', file, commitService, context);
}
await preflightWebZipImport(file);
const snapshot = await snapshotFile(file);
const { batch } = await BearTransformer.planBearBackup({
collection,
schema: getAFFiNEWorkspaceSchema(),
imported: snapshot,
extensions: getStoreManager().config.init().value.get('store'),
});
return commitService.commitBatch(batch);
}
private createCommitService(options: {
organize?: boolean;
explorerIcon?: boolean;
tag?: boolean;
}) {
return new ImportCommitService({
collection: this.workspaceService.workspace.docCollection,
schema: getAFFiNEWorkspaceSchema(),
extensions: getStoreManager().config.init().value.get('store'),
organizeService: options.organize ? this.organizeService : undefined,
explorerIconService: options.explorerIcon
? this.explorerIconService
: undefined,
tagService: options.tag ? this.tagService : undefined,
logger,
});
}
}
async function detectNotionZipFormat(file: File): Promise<'markdown' | 'html'> {
const unzip = new Unzip();
await unzip.load(file);
let hasHtml = false;
for (const entry of unzip) {
const lower = entry.path.toLowerCase();
if (lower.endsWith('.md')) return 'markdown';
if (lower.endsWith('.html') && !lower.endsWith('/index.html')) {
hasHtml = true;
}
}
if (hasHtml) return 'html';
throw new Error('No Notion Markdown or HTML pages found in the archive');
}
async function snapshotReadableFiles(files: File[]) {
const snapshots: File[] = [];
const warnings: ImportWarning[] = [];
for (const file of files) {
try {
snapshots.push(await snapshotFile(file));
} catch (error) {
const sourcePath = file.webkitRelativePath || file.name;
const reason = error instanceof Error ? error.message : String(error);
warnings.push({
code: 'file-unreadable',
message: `Skipped unreadable file: ${sourcePath}. ${reason}`,
sourcePath,
});
}
}
return { files: snapshots, warnings };
}
packages/frontend/core/src/modules/index.ts
+2
View File
@@ -34,6 +34,7 @@ import { configureFeatureFlagModule } from './feature-flag';
import { configureGlobalContextModule } from './global-context';
import { configureI18nModule } from './i18n';
import { configureIconPickerModule } from './icon-picker';
import { configureImportModule } from './import';
import { configureImportClipperModule } from './import-clipper';
import { configureImportTemplateModule } from './import-template';
import { configureIntegrationModule } from './integration';
@@ -103,6 +104,7 @@ export function configureCommonModules(framework: Framework) {
configureSystemFontFamilyModule(framework);
configureEditorSettingModule(framework);
configureImportTemplateModule(framework);
configureImportModule(framework);
configureUserspaceModule(framework);
configureAppSidebarModule(framework);
configureJournalModule(framework);
packages/frontend/electron-api/src/index.ts
+19 -3
View File
@@ -8,12 +8,18 @@ import type {
} from '@affine/electron/main/exposed';
import type { AppInfo } from '@affine/electron/preload/electron-api';
import type { SharedStorage } from '@affine/electron/preload/shared-storage';
import type { CreateImportSessionFromSourceOptions } from '@affine/electron/shared/import';
type MainHandlers = typeof mainHandlers;
type HelperHandlers = typeof helperHandlers;
type HelperEvents = typeof helperEvents;
type MainEvents = typeof mainEvents;
export type ClientHandler = {
type ImportPreloadHandlers = {
createImportSessionFromSource(
options: CreateImportSessionFromSourceOptions
): Promise<string>;
};
type MainClientHandlers = {
[namespace in keyof MainHandlers]: {
[method in keyof MainHandlers[namespace]]: MainHandlers[namespace][method] extends (
arg0: any,
@@ -26,7 +32,12 @@ export type ClientHandler = {
: Promise<ReturnType<MainHandlers[namespace][method]>>
: never;
};
} & HelperHandlers;
};
export type ClientHandler = Omit<MainClientHandlers, 'import'> &
HelperHandlers & {
import: MainClientHandlers['import'] & ImportPreloadHandlers;
};
export type ClientEvents = MainEvents & HelperEvents;
export const appInfo = (globalThis as any).__appInfo as AppInfo | null;
@@ -38,7 +49,6 @@ export const sharedStorage = (globalThis as any).__sharedStorage as
| undefined;
export type { AppInfo, SharedStorage };
export {
type SpellCheckStateSchema,
type TabViewsMetaSchema,
@@ -51,3 +61,9 @@ export type {
AddTabOption,
TabAction,
} from '@affine/electron/main/windows-manager';
export type {
CreateImportSessionFromSourceOptions,
NativeImportBrowserSource,
NativeImportFormat,
NativeImportSessionHandlers,
} from '@affine/electron/shared/import';
packages/frontend/i18n/src/i18n-completenesses.json
+5 -5
View File
@@ -3,25 +3,25 @@
"ca": 92,
"da": 4,
"de": 100,
"el-GR": 91,
"el-GR": 90,
"en": 100,
"es-AR": 91,
"es-CL": 92,
"es": 91,
"fa": 91,
"fa": 90,
"fr": 95,
"hi": 1,
"it": 92,
"ja": 91,
"ja": 90,
"kk": 98,
"ko": 91,
"nb-NO": 45,
"pl": 92,
"pt-BR": 91,
"pt-BR": 90,
"ru": 93,
"sv-SE": 91,
"tr": 98,
"uk": 91,
"uk": 90,
"ur": 98,
"zh-Hans": 100,
"zh-Hant": 93
packages/frontend/media-capture-playground/.gitignore
-2
View File
@@ -1,2 +0,0 @@
recordings
.env
packages/frontend/media-capture-playground/package.json
-38
View File
@@ -1,38 +0,0 @@
{
"name": "@affine/media-capture-playground",
"private": true,
"type": "module",
"version": "0.27.0",
"scripts": {
"dev:web": "affine bundle --dev",
"build:web": "affine bundle",
"dev:server": "node --env-file-if-exists=.env --watch server/main.ts"
},
"dependencies": {
"@affine/native": "workspace:*",
"@google/generative-ai": "^0.24.0",
"@types/express": "^5.0.0",
"@types/lodash-es": "^4.17.12",
"@types/multer": "^2.0.0",
"@types/react": "^19.0.8",
"@types/react-dom": "^19.0.3",
"chokidar": "^4.0.3",
"express": "^5.0.0",
"express-rate-limit": "^7.1.5",
"fs-extra": "^11.3.0",
"lodash-es": "^4.17.23",
"multer": "^2.2.0",
"react": "^19.2.1",
"react-dom": "^19.2.1",
"react-markdown": "^10.1.0",
"socket.io": "^4.8.3",
"socket.io-client": "^4.8.3",
"swr": "^2.3.7",
"tailwindcss": "^4.1.17"
},
"devDependencies": {
"@types/fs-extra": "^11",
"@types/react": "^19.0.1",
"@types/react-dom": "^19.0.2"
}
}
packages/frontend/media-capture-playground/server/encode.ts
-58
View File
@@ -1,58 +0,0 @@
export function createWavBuffer(
samples: Float32Array,
options: {
sampleRate?: number;
numChannels?: number;
}
) {
const { sampleRate = 44100, numChannels = 1 } = options;
const bitsPerSample = 16;
const bytesPerSample = bitsPerSample / 8;
const dataSize = samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataSize); // WAV header is 44 bytes
const view = new DataView(buffer);
// Write WAV header
// "RIFF" chunk descriptor
writeString(view, 0, 'RIFF');
view.setUint32(4, 36 + dataSize, true); // File size - 8
writeString(view, 8, 'WAVE');
// "fmt " sub-chunk
writeString(view, 12, 'fmt ');
view.setUint32(16, 16, true); // Sub-chunk size
view.setUint16(20, 1, true); // Audio format (1 = PCM)
view.setUint16(22, numChannels, true); // Channels
view.setUint32(24, sampleRate, true); // Sample rate
view.setUint32(28, sampleRate * numChannels * bytesPerSample, true); // Byte rate
view.setUint16(32, numChannels * bytesPerSample, true); // Block align
view.setUint16(34, bitsPerSample, true); // Bits per sample
// "data" sub-chunk
writeString(view, 36, 'data');
view.setUint32(40, dataSize, true); // Sub-chunk size
// Write audio data
const offset = 44;
for (let i = 0; i < samples.length; i++) {
// Convert float32 to int16
const s = Math.max(-1, Math.min(1, samples[i]));
view.setInt16(
offset + i * bytesPerSample,
s < 0 ? s * 0x8000 : s * 0x7fff,
true
);
}
return buffer;
}
function writeString(
view: DataView<ArrayBuffer>,
offset: number,
string: string
) {
for (let i = 0; i < string.length; i++) {
view.setUint8(offset + i, string.charCodeAt(i));
}
}
packages/frontend/media-capture-playground/server/gemini.ts
-200
View File
@@ -1,200 +0,0 @@
import { GoogleGenerativeAI } from '@google/generative-ai';
import {
GoogleAIFileManager,
type UploadFileResponse,
} from '@google/generative-ai/server';
const DEFAULT_MODEL = 'gemini-2.5-pro';
export interface TranscriptionResult {
title: string;
summary: string;
segments: {
speaker: string;
start_time: string;
end_time: string;
transcription: string;
}[];
}
const PROMPT_TRANSCRIPTION = `
Generate audio transcription and diarization for the recording.
The recording source is most likely from a video call with multiple speakers.
Output in JSON format with the following structure:
{
"segments": [
{
"speaker": "Speaker A",
"start_time": "MM:SS",
"end_time": "MM:SS",
"transcription": "..."
},
...
],
}
- Use consistent speaker labels throughout
- Accurate timestamps in MM:SS format
- Clean transcription with proper punctuation
- Identify speakers by name if possible, otherwise use "Speaker A/B/C"
`;
const PROMPT_SUMMARY = `
Generate a short title and summary of the conversation. The input is in the following JSON format:
{
"segments": [
{
"speaker": "Speaker A",
"start_time": "MM:SS",
"end_time": "MM:SS",
"transcription": "..."
},
...
],
}
Output in JSON format with the following structure:
{
"title": "Title of the recording",
"summary": "Summary of the conversation in markdown format"
}
1. Summary Structure:
- The sumary should be inferred from the speakers' language and context
- All insights should be derived directly from speakers' language and context
- Use hierarchical organization for clear information structure
- Use markdown format for the summary. Use bullet points, lists and other markdown styles when appropriate
2. Title:
- Come up with a title for the recording.
- The title should be a short description of the recording.
- The title should be a single sentence or a few words.
`;
export async function gemini(
audioFilePath: string,
options?: {
model?: 'gemini-2.5-flash' | 'gemini-2.5-pro';
mode?: 'transcript' | 'summary';
}
) {
if (!process.env.GOOGLE_GEMINI_API_KEY) {
console.error('Missing GOOGLE_GEMINI_API_KEY environment variable');
throw new Error('GOOGLE_GEMINI_API_KEY is not set');
}
// Initialize GoogleGenerativeAI and FileManager with your API_KEY
const genAI = new GoogleGenerativeAI(process.env.GOOGLE_GEMINI_API_KEY);
const fileManager = new GoogleAIFileManager(
process.env.GOOGLE_GEMINI_API_KEY
);
async function transcribe(
audioFilePath: string
): Promise<TranscriptionResult | null> {
let uploadResult: UploadFileResponse | null = null;
try {
// Upload the audio file
uploadResult = await fileManager.uploadFile(audioFilePath, {
mimeType: 'audio/wav',
displayName: 'audio_transcription.wav',
});
console.log('File uploaded:', uploadResult.file.uri);
// Initialize a Gemini model appropriate for your use case.
const model = genAI.getGenerativeModel({
model: options?.model || DEFAULT_MODEL,
generationConfig: {
responseMimeType: 'application/json',
},
});
// Generate content using a prompt and the uploaded file
const result = await model.generateContent([
{
fileData: {
fileUri: uploadResult.file.uri,
mimeType: uploadResult.file.mimeType,
},
},
{
text: PROMPT_TRANSCRIPTION,
},
]);
const text = result.response.text();
try {
const parsed = JSON.parse(text);
return parsed;
} catch (e) {
console.error('Failed to parse transcription JSON:', e);
console.error('Raw text that failed to parse:', text);
return null;
}
} catch (e) {
console.error('Error during transcription:', e);
return null;
} finally {
if (uploadResult) {
await fileManager.deleteFile(uploadResult.file.name);
}
}
}
async function summarize(transcription: TranscriptionResult) {
try {
const model = genAI.getGenerativeModel({
model: options?.model || DEFAULT_MODEL,
generationConfig: {
responseMimeType: 'application/json',
},
});
const result = await model.generateContent([
{
text: PROMPT_SUMMARY + '\n\n' + JSON.stringify(transcription),
},
]);
const text = result.response.text();
try {
const parsed = JSON.parse(text);
return parsed;
} catch (e) {
console.error('Failed to parse summary JSON:', e);
console.error('Raw text that failed to parse:', text);
return null;
}
} catch (e) {
console.error('Error during summarization:', e);
return null;
}
}
const transcription = await transcribe(audioFilePath);
if (!transcription) {
console.error('Transcription failed');
return null;
}
const summary = await summarize(transcription);
if (!summary) {
console.error('Summary generation failed');
return transcription;
}
const result = {
...transcription,
...summary,
};
console.log('Processing completed:', {
title: result.title,
segmentsCount: result.segments?.length,
});
return result;
}
packages/frontend/media-capture-playground/server/main.ts
-1078
View File
File diff suppressed because it is too large Load Diff
packages/frontend/media-capture-playground/server/types.d.ts
Vendored
-4
View File
@@ -1,4 +0,0 @@
declare module '*.txt' {
const content: string;
export default content;
}
packages/frontend/media-capture-playground/tsconfg.node.json
-7
View File
@@ -1,7 +0,0 @@
{
"extends": "../../../tsconfig.node.json",
"compilerOptions": {
"rootDir": "./server"
},
"include": ["./server"]
}
packages/frontend/media-capture-playground/tsconfig.json
-10
View File
@@ -1,10 +0,0 @@
{
"extends": "../../../tsconfig.web.json",
"compilerOptions": {
"rootDir": "./web",
"outDir": "./dist",
"tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo"
},
"include": ["./web", "server/types.d.ts"],
"references": [{ "path": "../native" }]
}
packages/frontend/media-capture-playground/web/app.tsx
-38
View File
@@ -1,38 +0,0 @@
import { AppList } from './components/app-list';
import { GlobalRecordButton } from './components/global-record-button';
import { SavedRecordings } from './components/saved-recordings';
export function App() {
return (
<div className="h-screen bg-gray-50 overflow-hidden">
<div className="h-full p-4 flex gap-4 max-w-[1800px] mx-auto">
<div className="flex-1 flex flex-col min-h-0">
<div className="flex justify-between items-center mb-1">
<h1 className="text-xl font-bold text-gray-900">
Running Applications
</h1>
<GlobalRecordButton />
</div>
<p className="text-sm text-gray-500 mb-2">
Select an application to start recording its audio, or use global
recording for system-wide audio
</p>
<div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 overflow-auto">
<AppList />
</div>
</div>
<div className="w-[1024px] flex flex-col min-h-0">
<h1 className="text-xl font-bold text-gray-900 mb-1">
Saved Recordings
</h1>
<p className="text-sm text-gray-500 mb-2">
Listen to and manage your recorded audio files
</p>
<div className="flex-1 bg-white shadow-lg rounded-lg border border-gray-100 p-4 overflow-auto">
<SavedRecordings />
</div>
</div>
</div>
</div>
);
}
packages/frontend/media-capture-playground/web/components/app-item.tsx
-122
View File
@@ -1,122 +0,0 @@
import React from 'react';
import type { AppGroup, RecordingStatus } from '../types';
import { formatDuration } from '../utils';
interface AppItemProps {
app: AppGroup;
recordings?: RecordingStatus[];
}
export function AppItem({ app, recordings }: AppItemProps) {
const [imgError, setImgError] = React.useState(false);
const [isRecording, setIsRecording] = React.useState(false);
const appName = app.rootApp.name || '';
const bundleId = app.rootApp.bundleIdentifier || '';
const firstLetter = appName.charAt(0).toUpperCase();
const isRunning = app.apps.some(a => a.isRunning);
const recording = recordings?.find((r: RecordingStatus) =>
app.apps.some(a => a.processId === r.processId)
);
const handleRecordClick = React.useCallback(() => {
const recordingApp = app.apps.find(a => a.isRunning);
if (!recordingApp) {
return;
}
if (isRecording) {
void fetch(`/api/apps/${recordingApp.processId}/stop`, {
method: 'POST',
})
.then(() => setIsRecording(false))
.catch(error => console.error('Failed to stop recording:', error));
} else {
void fetch(`/api/apps/${recordingApp.processId}/record`, {
method: 'POST',
})
.then(() => setIsRecording(true))
.catch(error => console.error('Failed to start recording:', error));
}
}, [app.apps, isRecording]);
React.useEffect(() => {
setIsRecording(!!recording);
}, [recording]);
const [duration, setDuration] = React.useState(0);
React.useEffect(() => {
if (recording) {
const interval = setInterval(() => {
setDuration(Date.now() - recording.startTime);
}, 1000);
return () => clearInterval(interval);
} else {
setDuration(0);
}
return () => {};
}, [recording]);
return (
<div className="flex items-center h-16 space-x-2 p-3 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100">
{imgError ? (
<div className="w-8 h-8 rounded-lg bg-gray-50 border border-gray-100 flex items-center justify-center text-gray-600 font-semibold text-base">
{firstLetter}
</div>
) : (
<img
src={`/api/apps/${app.rootApp.processId}/icon`}
loading="lazy"
alt={appName}
className="w-8 h-8 object-contain rounded-lg bg-gray-50 border border-gray-100"
onError={() => setImgError(true)}
/>
)}
<div className="flex-1 min-w-0">
<div className="flex items-center space-x-1 mb-1">
{appName ? (
<span className="text-gray-900 font-medium text-sm truncate">
{appName}
</span>
) : (
<span className="text-gray-400 italic font-medium text-sm">
Unnamed Application
</span>
)}
<span className="text-xs px-1 bg-gray-50 text-gray-500 rounded border border-gray-100">
PID: {app.rootApp.processId}
</span>
<span
className={`text-xs px-2 py-0.5 rounded-full font-medium border ${recording ? 'bg-red-50 text-red-600 border-red-100 opacity-100' : 'opacity-0'}`}
>
{recording ? formatDuration(duration) : '00:00:00'}
</span>
</div>
<div className="text-xs text-gray-500 font-mono truncate opacity-80">
{bundleId}
</div>
</div>
{(isRunning || isRecording) && (
<button
onClick={handleRecordClick}
className={`h-8 min-w-[80px] flex items-center justify-center rounded-lg text-sm font-medium transition-all duration-200 ${
isRecording
? 'bg-red-50 text-red-600 hover:bg-red-100 border border-red-200'
: 'bg-blue-50 text-blue-600 hover:bg-blue-100 border border-blue-200'
}`}
>
{isRecording ? (
<>
<div className="w-1.5 h-1.5 rounded-full bg-red-500 animate-pulse mr-2" />
<span>Stop</span>
</>
) : (
<span>Record</span>
)}
</button>
)}
</div>
);
}
packages/frontend/media-capture-playground/web/components/app-list.tsx
-145
View File
@@ -1,145 +0,0 @@
import React from 'react';
import useSWRSubscription from 'swr/subscription';
import type { App, AppGroup, RecordingStatus } from '../types';
import { socket } from '../utils';
import { AppItem } from './app-item';
export function AppList() {
const { data: apps = [] } = useSWRSubscription('apps', (_key, { next }) => {
let apps: App[] = [];
// Initial apps fetch
fetch('/api/apps')
.then(res => res.json())
.then(data => {
apps = data.apps;
next(null, apps);
})
.catch(err => next(err));
// Subscribe to app updates
socket.on('apps:all', data => {
next(null, data.apps);
apps = data.apps;
});
socket.on('apps:state-changed', data => {
const index = apps.findIndex(a => a.processId === data.processId);
console.log('apps:state-changed', data, index);
if (index !== -1) {
next(
null,
apps.toSpliced(index, 1, {
...apps[index],
isRunning: data.isRunning,
})
);
}
});
socket.on('connect', () => {
// Refetch on reconnect
fetch('/api/apps')
.then(res => res.json())
.then(data => next(null, data.apps))
.catch(err => next(err));
});
return () => {
socket.off('apps:all');
socket.off('apps:state-changed');
socket.off('connect');
};
});
const { data: recordings = [] } = useSWRSubscription<RecordingStatus[]>(
'recordings',
(
_key: string,
{ next }: { next: (err: Error | null, data?: RecordingStatus[]) => void }
) => {
// Subscribe to recording updates
socket.on('apps:recording', (data: { recordings: RecordingStatus[] }) => {
next(null, data.recordings);
});
return () => {
socket.off('apps:recording');
};
}
);
const appGroups: AppGroup[] = React.useMemo(() => {
const mapping = apps.reduce((acc: Record<number, AppGroup>, app: App) => {
if (!acc[app.processGroupId]) {
acc[app.processGroupId] = {
processGroupId: app.processGroupId,
apps: [],
rootApp:
apps.find((a: App) => a.processId === app.processGroupId) || app,
};
}
acc[app.processGroupId].apps.push(app);
return acc;
}, {});
return Object.values(mapping);
}, [apps]);
const runningApps = (appGroups || []).filter(app =>
app.apps.some(a => a.isRunning)
);
const notRunningApps = (appGroups || []).filter(
app => !app.apps.some(a => a.isRunning)
);
return (
<div className="h-full flex flex-col divide-y divide-gray-100">
<div className="p-4 relative">
<div className="flex items-center justify-between sticky top-0 bg-white z-10 mb-2">
<h2 className="text-sm font-semibold text-gray-900">
Active Applications
</h2>
<span className="text-xs px-2 py-1 bg-blue-50 rounded-full text-blue-600 font-medium">
{runningApps.length} listening
</span>
</div>
<div className="space-y-2">
{runningApps.map(app => (
<AppItem
key={app.processGroupId}
app={app}
recordings={recordings}
/>
))}
{runningApps.length === 0 && (
<div className="text-sm text-gray-500 italic bg-gray-50 rounded-xl p-4 text-center">
No applications are currently listening
</div>
)}
</div>
</div>
<div className="p-4 flex-1 relative">
<div className="flex items-center justify-between sticky top-0 bg-white z-10 mb-2">
<h2 className="text-sm font-semibold text-gray-900">
Other Applications
</h2>
<span className="text-xs px-2 py-1 bg-gray-50 rounded-full text-gray-600 font-medium">
{notRunningApps.length} available
</span>
</div>
<div className="space-y-2">
{notRunningApps.map(app => (
<AppItem
key={app.processGroupId}
app={app}
recordings={recordings}
/>
))}
{notRunningApps.length === 0 && (
<div className="text-sm text-gray-500 italic bg-gray-50 rounded-xl p-4 text-center">
No other applications found
</div>
)}
</div>
</div>
</div>
);
}
packages/frontend/media-capture-playground/web/components/global-record-button.tsx
-70
View File
@@ -1,70 +0,0 @@
import { useCallback, useEffect, useState } from 'react';
import { socket } from '../utils';
export function GlobalRecordButton() {
const [isRecording, setIsRecording] = useState(false);
const [isLoading, setIsLoading] = useState(false);
useEffect(() => {
function handleRecordingStatus(data: {
recordings: Array<{ processId: number }>;
}) {
// Global recording uses processId -1
setIsRecording(data.recordings.some(r => r.processId === -1));
}
socket.on('apps:recording', handleRecordingStatus);
return () => {
socket.off('apps:recording', handleRecordingStatus);
};
}, []);
const handleClick = useCallback(() => {
setIsLoading(true);
const endpoint = isRecording ? '/api/global/stop' : '/api/global/record';
fetch(endpoint, { method: 'POST' })
.then(response => {
if (!response.ok) {
throw new Error('Failed to toggle global recording');
}
})
.catch(error => {
console.error('Error toggling global recording:', error);
})
.finally(() => {
setIsLoading(false);
});
}, [isRecording]);
return (
<button
onClick={handleClick}
disabled={isLoading}
className={`
px-4 py-2 rounded-lg font-medium text-sm
transition-colors duration-200
${isLoading ? 'opacity-50 cursor-not-allowed' : ''}
${
isRecording
? 'bg-red-100 text-red-700 hover:bg-red-200'
: 'bg-blue-100 text-blue-700 hover:bg-blue-200'
}
`}
>
<div className="flex items-center gap-2">
{isRecording ? (
<>
<div className="w-2 h-2 rounded-full bg-red-500 animate-pulse" />
Stop Global Recording
</>
) : (
<>
<div className="w-2 h-2 rounded-full bg-blue-500" />
Record System Audio
</>
)}
</div>
</button>
);
}
packages/frontend/media-capture-playground/web/components/icons.tsx
-163
View File
@@ -1,163 +0,0 @@
import type { ReactElement } from 'react';
export function PlayIcon(): ReactElement {
return (
<svg
className="w-6 h-6 text-gray-900"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M4.5 5.653c0-1.426 1.529-2.33 2.779-1.643l11.54 6.348c1.295.712 1.295 2.573 0 3.285L7.28 19.991c-1.25.687-2.779-.217-2.779-1.643V5.653z"
fill="currentColor"
/>
</svg>
);
}
export function PauseIcon(): ReactElement {
return (
<svg
className="w-6 h-6 text-gray-900"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
fillRule="evenodd"
clipRule="evenodd"
d="M6.75 5.25a.75.75 0 01.75-.75H9a.75.75 0 01.75.75v13.5a.75.75 0 01-.75.75H7.5a.75.75 0 01-.75-.75V5.25zm7 0a.75.75 0 01.75-.75h1.5a.75.75 0 01.75.75v13.5a.75.75 0 01-.75.75h-1.5a.75.75 0 01-.75-.75V5.25z"
fill="currentColor"
/>
</svg>
);
}
export function RewindIcon(): ReactElement {
return (
<svg
className="w-5 h-5 text-gray-600"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M12.066 11.2a1 1 0 000 1.6l5.334 4A1 1 0 0019 16V8a1 1 0 00-1.6-.8l-5.334 4zM11 8a1 1 0 00-1.6-.8l-5.334 4a1 1 0 000 1.6l5.334 4A1 1 0 0011 16V8z"
fill="currentColor"
/>
</svg>
);
}
export function ForwardIcon(): ReactElement {
return (
<svg
className="w-5 h-5 text-gray-600"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M5 8a1 1 0 011.6-.8l5.334 4a1 1 0 010 1.6L6.6 16.8A1 1 0 015 16V8zm7.066-.8a1 1 0 00-1.6.8v8a1 1 0 001.6.8l5.334-4a1 1 0 000-1.6l-5.334-4z"
fill="currentColor"
/>
</svg>
);
}
export function DeleteIcon(): ReactElement {
return (
<svg
className="w-5 h-5"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
/>
</svg>
);
}
export function LoadingSpinner(): ReactElement {
return (
<svg className="animate-spin h-4 w-4" viewBox="0 0 24 24">
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
fill="none"
/>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
/>
</svg>
);
}
export function ErrorIcon(): ReactElement {
return (
<svg
className="w-4 h-4 mr-1.5 flex-shrink-0"
fill="currentColor"
viewBox="0 0 20 20"
>
<path
fillRule="evenodd"
d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
clipRule="evenodd"
/>
</svg>
);
}
export function MicrophoneIcon(): ReactElement {
return (
<svg
className="w-4 h-4 mr-1.5 text-blue-500"
viewBox="0 0 20 20"
fill="currentColor"
>
<path d="M7 4a3 3 0 016 0v4a3 3 0 11-6 0V4zm4 10.93A7.001 7.001 0 0017 8a1 1 0 10-2 0A5 5 0 015 8a1 1 0 00-2 0 7.001 7.001 0 006 6.93V17H6a1 1 0 100 2h8a1 1 0 100-2h-3v-2.07z" />
</svg>
);
}
export function WarningIcon(): ReactElement {
return (
<svg className="w-4 h-4 mr-1.5" viewBox="0 0 20 20" fill="currentColor">
<path
fillRule="evenodd"
d="M8.257 3.099c.765-1.36 2.722-1.36 3.486 0l5.58 9.92c.75 1.334-.213 2.98-1.742 2.98H4.42c-1.53 0-2.493-1.646-1.743-2.98l5.58-9.92zM11 13a1 1 0 11-2 0 1 1 0 012 0zm-1-8a1 1 0 00-1 1v3a1 1 0 002 0V6a1 1 0 00-1-1z"
clipRule="evenodd"
/>
</svg>
);
}
export function DefaultAppIcon(): ReactElement {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
className="h-6 w-6"
viewBox="0 0 20 20"
fill="currentColor"
>
<path d="M10 2a3 3 0 00-3 3v4a3 3 0 006 0V5a3 3 0 00-3-3zm0 2a1 1 0 011 1v4a1 1 0 11-2 0V5a1 1 0 011-1z" />
<path d="M3 10a7 7 0 1014 0h-2a5 5 0 11-10 0H3z" />
</svg>
);
}
packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx
-895
View File
@@ -1,895 +0,0 @@
import type { ReactElement } from 'react';
import React from 'react';
import ReactMarkdown from 'react-markdown';
import type { SavedRecording, TranscriptionMetadata } from '../types';
import { formatDuration, socket } from '../utils';
import {
DefaultAppIcon,
DeleteIcon,
ErrorIcon,
ForwardIcon,
LoadingSpinner,
MicrophoneIcon,
PauseIcon,
PlayIcon,
RewindIcon,
WarningIcon,
} from './icons';
interface SavedRecordingItemProps {
recording: SavedRecording;
}
// Audio player controls component
function AudioControls({
audioRef,
playbackRate,
onPlaybackRateChange,
onSeek,
onPlayPause,
}: {
audioRef: React.RefObject<HTMLAudioElement | null>;
playbackRate: number;
onPlaybackRateChange: () => void;
onSeek: (seconds: number) => void;
onPlayPause: () => void;
}): ReactElement {
const [currentTime, setCurrentTime] = React.useState('00:00');
const [duration, setDuration] = React.useState('00:00');
React.useEffect(() => {
const audio = audioRef.current;
if (!audio) return;
const formatTime = (time: number) => {
const minutes = Math.floor(time / 60);
const seconds = Math.floor(time % 60);
return `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`;
};
const updateTime = () => {
setCurrentTime(formatTime(audio.currentTime));
setDuration(formatTime(audio.duration));
};
audio.addEventListener('timeupdate', updateTime);
audio.addEventListener('loadedmetadata', updateTime);
return () => {
audio.removeEventListener('timeupdate', updateTime);
audio.removeEventListener('loadedmetadata', updateTime);
};
}, [audioRef]);
return (
<div className="flex items-center justify-between">
<div className="flex items-center space-x-2">
<button
onClick={() => onSeek(-15)}
className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"
title="Back 15 seconds"
>
<RewindIcon />
</button>
<button
onClick={onPlayPause}
className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"
>
{audioRef.current?.paused ? <PlayIcon /> : <PauseIcon />}
</button>
<button
onClick={() => onSeek(30)}
className="p-2 hover:bg-gray-50 rounded-lg transition-all duration-200 border border-transparent hover:border-gray-100 hover:shadow-sm"
title="Forward 30 seconds"
>
<ForwardIcon />
</button>
<div className="text-sm font-mono text-gray-500 ml-2">
{currentTime} <span className="text-gray-400">/</span> {duration}
</div>
</div>
<button
onClick={onPlaybackRateChange}
className="px-3 py-1.5 text-sm font-medium text-gray-600 bg-gray-50 hover:bg-gray-100 rounded-lg transition-all duration-200 border border-gray-100 hover:shadow-sm"
>
{playbackRate}x
</button>
</div>
);
}
// Waveform visualization component
function WaveformVisualizer({
containerRef,
waveformData,
currentTime,
fileName,
}: {
containerRef: React.RefObject<HTMLDivElement | null>;
waveformData: number[];
currentTime: number;
fileName: string;
}): ReactElement {
return (
<div
className="relative h-14 bg-gray-50 overflow-hidden rounded-lg border border-gray-100"
ref={containerRef}
>
<div className="absolute inset-0 flex items-end">
{waveformData.map((amplitude, i) => (
<div
key={`${fileName}-bar-${i}`}
className="flex-1 bg-red-400 transition-all duration-200"
style={{
height: `${Math.max(amplitude * 100, 3)}%`,
opacity:
i < Math.floor(currentTime * waveformData.length) ? 1 : 0.3,
margin: '0 0.5px',
}}
/>
))}
</div>
</div>
);
}
// Update TranscriptionMessage component
function TranscriptionMessage({
item,
isNewSpeaker,
isCurrentMessage,
}: {
item: {
speaker: string;
start_time: string;
transcription: string;
};
isNewSpeaker: boolean;
isCurrentMessage: boolean;
}): ReactElement {
return (
<div className="flex items-start gap-3 group transition-all duration-300 w-full">
<div className="w-[120px] flex-shrink-0">
<div className="flex flex-col items-start gap-1">
{isNewSpeaker && (
<div
className={`px-2.5 py-1 rounded-lg text-xs font-medium border transition-colors duration-300 ${
isCurrentMessage
? 'bg-blue-100 text-blue-700 border-blue-200'
: 'bg-blue-50 text-blue-600 border-blue-100'
}`}
>
{item.speaker}
</div>
)}
<div
className={`text-[11px] font-mono ml-2 transition-colors duration-300 ${
isCurrentMessage ? 'text-blue-500' : 'text-gray-400'
}`}
>
{item.start_time}
</div>
</div>
</div>
<div className="flex-1 min-w-0 w-full">
<div
className={`text-sm leading-relaxed rounded-xl px-4 py-2 border transition-all inline-flex duration-300 ${
isCurrentMessage
? 'bg-blue-50/50 text-blue-900 border-blue-200 shadow-md'
: 'bg-white text-gray-600 border-gray-100 shadow-sm hover:shadow-md'
}`}
>
{item.transcription}
</div>
</div>
</div>
);
}
// Add new Summary component
function TranscriptionSummary({ summary }: { summary: string }): ReactElement {
return (
<div className="mb-6 bg-blue-50/50 rounded-xl p-4 border border-blue-100">
<div className="text-xs font-medium text-blue-600 mb-2 uppercase tracking-wider">
Summary
</div>
<div className="text-sm text-gray-700 leading-relaxed prose prose-sm max-w-none prose-headings:text-gray-900 prose-a:text-blue-600 whitespace-pre-wrap">
<ReactMarkdown>{summary}</ReactMarkdown>
</div>
</div>
);
}
// Update TranscriptionContent component
function TranscriptionContent({
transcriptionData,
currentAudioTime,
}: {
transcriptionData: {
segments: Array<{
speaker: string;
start_time: string;
transcription: string;
}>;
summary: string;
title: string;
};
currentAudioTime: number;
}): ReactElement {
const parseTimestamp = (timestamp: string) => {
// Handle "MM:SS" format (without hours)
const [minutes, seconds] = timestamp.split(':');
return parseInt(minutes, 10) * 60 + parseInt(seconds, 10);
};
return (
<div className="space-y-2 py-2 max-h-[400px] overflow-y-auto pr-2 scrollbar-thin scrollbar-thumb-gray-300 scrollbar-track-transparent hover:scrollbar-thumb-gray-400 w-full">
<TranscriptionSummary summary={transcriptionData.summary} />
{transcriptionData.segments.map((item, index) => {
const isNewSpeaker =
index === 0 ||
transcriptionData.segments[index - 1].speaker !== item.speaker;
const startTime = parseTimestamp(item.start_time);
// Use next segment's start time as end time, or add 3 seconds for the last segment
const endTime =
index < transcriptionData.segments.length - 1
? parseTimestamp(transcriptionData.segments[index + 1].start_time)
: startTime + 3;
const isCurrentMessage =
currentAudioTime >= startTime && currentAudioTime < endTime;
return (
<TranscriptionMessage
key={`${item.speaker}-${item.start_time}-${index}`}
item={item}
isNewSpeaker={isNewSpeaker}
isCurrentMessage={isCurrentMessage}
/>
);
})}
</div>
);
}
// Update TranscriptionStatus component
function TranscriptionStatus({
transcription,
transcriptionError,
currentAudioTime,
}: {
transcription?: TranscriptionMetadata;
transcriptionError: string | null;
currentAudioTime: number;
}): ReactElement | null {
if (!transcription && !transcriptionError) {
return null;
}
if (transcription?.transcriptionStatus === 'pending') {
return (
<div className="my-2">
<div className="text-sm text-gray-600 bg-gray-50/50 p-4 border border-gray-100 w-full">
<div className="font-medium text-gray-900 mb-4 flex items-center sticky top-0 bg-gray-50/50 backdrop-blur-sm z-10 py-2">
<MicrophoneIcon />
<span>Processing Audio</span>
</div>
<div className="flex items-center justify-center py-8">
<div className="flex flex-col items-center gap-3">
<LoadingSpinner />
<div className="text-sm text-gray-600">
<span className="font-medium">Starting transcription</span>
<span className="text-gray-400 animate-pulse">...</span>
</div>
<div className="text-xs text-gray-400 max-w-sm text-center">
This may take a few moments depending on the length of the
recording
</div>
</div>
</div>
</div>
</div>
);
}
if (transcriptionError) {
return (
<div className="text-xs text-red-500 m-2 flex items-center bg-red-50 rounded-lg p-2 border border-red-100">
<ErrorIcon />
{transcriptionError}
</div>
);
}
if (
transcription?.transcriptionStatus === 'completed' &&
transcription.transcription
) {
try {
const transcriptionData = transcription.transcription;
if (
!transcriptionData.segments ||
!Array.isArray(transcriptionData.segments)
) {
throw new Error('Invalid transcription data format');
}
return (
<div className="my-2">
<div className="text-sm text-gray-600 bg-gray-50/50 p-4 border border-gray-100 w-full">
<div className="font-medium text-gray-900 mb-4 flex items-center sticky top-0 bg-gray-50/50 backdrop-blur-sm z-10 py-2">
<MicrophoneIcon />
<span>Conversation Transcript</span>
</div>
{transcriptionData.title && (
<div className="mb-4 bg-blue-50/50 rounded-lg p-3 border border-blue-100">
<div className="text-xs font-medium text-blue-600 uppercase tracking-wider mb-1">
Title
</div>
<div className="text-base font-medium text-gray-900">
{transcriptionData.title}
</div>
</div>
)}
<TranscriptionContent
transcriptionData={transcriptionData}
currentAudioTime={currentAudioTime}
/>
</div>
</div>
);
} catch (error) {
return (
<div className="text-sm text-red-500 bg-red-50 rounded-lg p-2 border border-red-100 m-2">
{error instanceof Error
? error.message
: 'Failed to parse transcription data'}
</div>
);
}
}
return null;
}
// Add new RecordingHeader component
function RecordingHeader({
metadata,
fileName,
recordingDate,
duration,
error,
isDeleting,
showDeleteConfirm,
setShowDeleteConfirm,
handleDeleteClick,
}: {
metadata: SavedRecording['metadata'];
fileName: string;
recordingDate: string;
duration: string;
error: string | null;
isDeleting: boolean;
showDeleteConfirm: boolean;
setShowDeleteConfirm: (show: boolean) => void;
handleDeleteClick: () => void;
transcriptionError: string | null;
}): ReactElement {
const [imgError, setImgError] = React.useState(false);
const isGlobalRecording = metadata?.isGlobal;
return (
<div className="flex items-start space-x-4 p-4 bg-gray-50/30">
<div className="relative w-12 h-12 flex-shrink-0">
{!imgError && !isGlobalRecording ? (
<img
src={`/api/recordings/${fileName}/icon.png`}
alt={metadata?.appName || 'Unknown Application'}
className="w-12 h-12 object-contain rounded-lg bg-gray-50 border border-gray-100 shadow-sm transition-transform duration-200 hover:scale-105"
onError={() => setImgError(true)}
/>
) : (
<div className="w-12 h-12 rounded-xl flex items-center justify-center text-gray-500 bg-gray-50 border border-gray-100 shadow-sm">
<DefaultAppIcon />
</div>
)}
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center justify-between">
<div className="flex items-center space-x-2">
<span className="text-gray-900 font-semibold text-base truncate">
{metadata?.appName || 'Unknown Application'}
</span>
{isGlobalRecording && (
<span className="text-xs px-2 py-0.5 bg-blue-50 rounded-full text-blue-600 font-medium border border-blue-100">
System Audio
</span>
)}
<span className="text-xs px-2 py-0.5 bg-gray-50 rounded-full text-gray-600 font-medium border border-gray-100">
{duration}
</span>
</div>
<div className="flex items-center">
{showDeleteConfirm ? (
<div className="flex items-center space-x-2">
<button
onClick={() => setShowDeleteConfirm(false)}
className="h-8 px-3 text-sm font-medium text-gray-600 hover:bg-gray-50 rounded-lg transition-colors border border-gray-100"
disabled={isDeleting}
>
Cancel
</button>
<button
onClick={handleDeleteClick}
className="h-8 px-3 text-sm font-medium text-red-600 hover:bg-red-50 rounded-lg transition-colors border border-red-100 disabled:opacity-50 disabled:cursor-not-allowed"
disabled={isDeleting}
>
{isDeleting ? (
<div className="flex items-center space-x-2">
<LoadingSpinner />
<span>Deleting...</span>
</div>
) : (
'Confirm'
)}
</button>
</div>
) : (
<button
onClick={() => setShowDeleteConfirm(true)}
className="h-8 w-8 flex items-center justify-center text-gray-400 hover:text-red-500 hover:bg-red-50 rounded-lg transition-colors"
title="Delete recording"
>
<DeleteIcon />
</button>
)}
</div>
</div>
<div className="text-sm text-gray-600 mt-1">{recordingDate}</div>
<div className="text-xs text-gray-400 font-mono mt-0.5 truncate">
{metadata?.bundleIdentifier || fileName}
</div>
{error && (
<div className="text-xs text-red-500 mt-2 flex items-center bg-red-50 rounded-lg p-2 border border-red-100">
<ErrorIcon />
{error}
</div>
)}
</div>
</div>
);
}
// Add new AudioPlayer component
function AudioPlayer({
isLoading,
error,
audioRef,
playbackRate,
handlePlaybackRateChange,
handleSeek,
handlePlayPause,
containerRef,
waveformData,
currentTime,
fileName,
}: {
isLoading: boolean;
error: string | null;
audioRef: React.RefObject<HTMLAudioElement>;
playbackRate: number;
handlePlaybackRateChange: () => void;
handleSeek: (seconds: number) => void;
handlePlayPause: () => void;
containerRef: React.RefObject<HTMLDivElement>;
waveformData: number[];
currentTime: number;
fileName: string;
}): ReactElement {
return (
<div className="px-4 pb-4">
{isLoading && !error ? (
<div className="h-14 bg-gray-50 rounded-lg flex items-center justify-center border border-gray-100">
<LoadingSpinner />
<span className="ml-2 text-sm text-gray-600 font-medium">
Loading audio...
</span>
</div>
) : (
<div className="flex flex-col space-y-3">
<AudioControls
audioRef={audioRef}
playbackRate={playbackRate}
onPlaybackRateChange={handlePlaybackRateChange}
onSeek={handleSeek}
onPlayPause={handlePlayPause}
/>
<WaveformVisualizer
containerRef={containerRef}
waveformData={waveformData}
currentTime={currentTime}
fileName={fileName}
/>
</div>
)}
</div>
);
}
// Add new TranscribeButton component
function TranscribeButton({
transcriptionStatus,
onTranscribe,
}: {
transcriptionStatus?: TranscriptionMetadata['transcriptionStatus'];
onTranscribe: () => void;
}): ReactElement {
return (
<div className="px-4 pb-4">
<div className="flex justify-end">
<button
onClick={onTranscribe}
disabled={transcriptionStatus === 'pending'}
className={`h-8 px-3 text-sm font-medium rounded-lg transition-colors border flex items-center space-x-2
${
transcriptionStatus === 'pending'
? 'bg-blue-50 text-blue-600 border-blue-200 cursor-not-allowed'
: transcriptionStatus === 'completed'
? 'text-blue-600 hover:bg-blue-50 border-blue-100'
: transcriptionStatus === 'error'
? 'text-red-600 hover:bg-red-50 border-red-100'
: 'text-blue-600 hover:bg-blue-50 border-blue-100'
}`}
>
{transcriptionStatus === 'pending' ? (
<>
<LoadingSpinner />
<span>Transcribing...</span>
</>
) : transcriptionStatus === 'completed' ? (
<>
<MicrophoneIcon />
<span>Transcribe Again</span>
</>
) : transcriptionStatus === 'error' ? (
<>
<WarningIcon />
<span>Retry Transcription</span>
</>
) : (
<>
<MicrophoneIcon />
<span>Transcribe</span>
</>
)}
</button>
</div>
</div>
);
}
// Main SavedRecordingItem component (simplified)
export function SavedRecordingItem({
recording,
}: SavedRecordingItemProps): ReactElement {
const [error, setError] = React.useState<string | null>(null);
const [isLoading, setIsLoading] = React.useState(true);
const [isDeleting, setIsDeleting] = React.useState(false);
const [showDeleteConfirm, setShowDeleteConfirm] = React.useState(false);
const [playbackRate, setPlaybackRate] = React.useState(1);
const [waveformData, setWaveformData] = React.useState<number[]>([]);
const [currentTime, setCurrentTime] = React.useState(0);
const audioRef = React.useRef<HTMLAudioElement | null>(null);
const containerRef = React.useRef<HTMLDivElement | null>(null);
const [segments, setSegments] = React.useState(40);
const [currentAudioTime, setCurrentAudioTime] = React.useState(0);
const [transcriptionError, setTranscriptionError] = React.useState<
string | null
>(null);
const metadata = recording.metadata;
// Ensure we have a valid filename, fallback to an empty string if undefined
const fileName = recording.wav || '';
const recordingDate = metadata
? new Date(metadata.recordingStartTime).toLocaleString()
: 'Unknown date';
const duration = metadata
? formatDuration(metadata.recordingDuration * 1000)
: 'Unknown duration';
// Update current audio time
React.useEffect(() => {
const audio = audioRef.current;
if (audio) {
const handleTimeUpdate = () => {
setCurrentAudioTime(audio.currentTime);
};
audio.addEventListener('timeupdate', handleTimeUpdate);
return () => audio.removeEventListener('timeupdate', handleTimeUpdate);
}
return () => {};
}, []);
// Calculate number of segments based on container width
React.useEffect(() => {
const updateSegments = () => {
if (containerRef.current) {
// Each bar should be at least 2px wide (1px bar + 1px gap)
const width = containerRef.current.offsetWidth;
setSegments(Math.floor(width / 2));
}
};
updateSegments();
const resizeObserver = new ResizeObserver(updateSegments);
if (containerRef.current) {
resizeObserver.observe(containerRef.current);
}
return () => resizeObserver.disconnect();
}, []);
const processAudioData = React.useCallback(async () => {
try {
// Check if fileName is empty
if (!fileName) {
throw new Error('Invalid recording filename');
}
const response = await fetch(`/api/recordings/${fileName}/recording.wav`);
if (!response.ok) {
throw new Error(
`Failed to fetch audio file (${response.status}): ${response.statusText}`
);
}
const audioContext = new AudioContext();
const arrayBuffer = await response.arrayBuffer();
// Ensure we have data to process
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
throw new Error('No audio data received');
}
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
const channelData = audioBuffer.getChannelData(0);
// Process the audio data in chunks to create the waveform
const numberOfSamples = channelData.length;
const samplesPerSegment = Math.floor(numberOfSamples / segments);
const waveform: number[] = [];
for (let i = 0; i < segments; i++) {
const start = i * samplesPerSegment;
const end = start + samplesPerSegment;
const segmentData = channelData.slice(start, end);
// Calculate RMS (root mean square) for better amplitude representation
const rms = Math.sqrt(
segmentData.reduce((sum, sample) => sum + sample * sample, 0) /
segmentData.length
);
waveform.push(rms);
}
// Normalize the waveform data to a 0-1 range
const maxAmplitude = Math.max(...waveform);
const normalizedWaveform = waveform.map(amp => amp / maxAmplitude);
setWaveformData(normalizedWaveform);
setIsLoading(false);
} catch (err) {
console.error('Error processing audio:', err);
setError(
err instanceof Error ? err.message : 'Failed to process audio data'
);
setIsLoading(false);
}
}, [fileName, segments]);
React.useEffect(() => {
const audio = audioRef.current;
if (audio) {
const handleError = (e: ErrorEvent) => {
console.error('Audio error:', e);
setError('Failed to load audio');
setIsLoading(false);
};
const handleLoadedMetadata = () => {
void processAudioData().catch(err => {
console.error('Error processing audio data:', err);
setError('Failed to process audio data');
setIsLoading(false);
});
};
const handleTimeUpdate = () => {
setCurrentTime(audio.currentTime / audio.duration);
};
audio.addEventListener('error', handleError as EventListener);
audio.addEventListener('loadedmetadata', handleLoadedMetadata);
audio.addEventListener('timeupdate', handleTimeUpdate);
return () => {
audio.removeEventListener('error', handleError as EventListener);
audio.removeEventListener('loadedmetadata', handleLoadedMetadata);
audio.removeEventListener('timeupdate', handleTimeUpdate);
};
}
return () => {};
}, [processAudioData]);
const handlePlayPause = React.useCallback(() => {
if (audioRef.current) {
if (audioRef.current.paused) {
// oxlint-disable-next-line @typescript-eslint/no-floating-promises
void audioRef.current.play();
} else {
audioRef.current.pause();
}
}
}, []);
const handleSeek = React.useCallback((seconds: number) => {
if (audioRef.current) {
audioRef.current.currentTime += seconds;
}
}, []);
const handlePlaybackRateChange = React.useCallback(() => {
if (audioRef.current) {
const newRate = playbackRate === 1 ? 1.5 : 1;
audioRef.current.playbackRate = newRate;
setPlaybackRate(newRate);
}
}, [playbackRate]);
const handleDelete = React.useCallback(async () => {
setIsDeleting(true);
setError(null); // Clear any previous errors
try {
// Check if filename is valid
if (!recording.wav) {
throw new Error('Invalid recording filename');
}
const response = await fetch(`/api/recordings/${recording.wav}`, {
method: 'DELETE',
});
if (!response.ok) {
let errorMessage: string;
try {
const errorData = await response.json();
errorMessage = errorData.error;
} catch {
errorMessage = `Server error (${response.status}): ${response.statusText}`;
}
throw new Error(errorMessage);
}
setShowDeleteConfirm(false);
} catch (err) {
console.error('Error deleting recording:', err);
setError(
err instanceof Error ? err.message : 'An unexpected error occurred'
);
} finally {
setIsDeleting(false);
}
}, [recording.wav]);
const handleDeleteClick = React.useCallback(() => {
void handleDelete().catch(err => {
console.error('Unexpected error during deletion:', err);
setError('An unexpected error occurred');
});
}, [handleDelete]);
React.useEffect(() => {
// Listen for transcription events
socket.on(
'apps:recording-transcription-start',
(data: { filename: string }) => {
if (recording.wav && data.filename === recording.wav) {
setTranscriptionError(null);
}
}
);
socket.on(
'apps:recording-transcription-end',
(data: {
filename: string;
success: boolean;
transcription?: string;
error?: string;
}) => {
if (recording.wav && data.filename === recording.wav && !data.success) {
setTranscriptionError(data.error || 'Transcription failed');
}
}
);
return () => {
socket.off('apps:recording-transcription-start');
socket.off('apps:recording-transcription-end');
};
}, [recording.wav]);
const handleTranscribe = React.useCallback(async () => {
try {
// Check if filename is valid
if (!recording.wav) {
throw new Error('Invalid recording filename');
}
const response = await fetch(
`/api/recordings/${recording.wav}/transcribe`,
{
method: 'POST',
}
);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to start transcription');
}
} catch (err) {
setTranscriptionError(
err instanceof Error ? err.message : 'Failed to start transcription'
);
}
}, [recording.wav]);
return (
<div className="bg-white rounded-lg shadow-sm hover:shadow-md transition-all duration-300 overflow-hidden mb-3 border border-gray-100 hover:border-gray-200">
<RecordingHeader
metadata={metadata}
fileName={fileName}
recordingDate={recordingDate}
duration={duration}
error={error}
isDeleting={isDeleting}
showDeleteConfirm={showDeleteConfirm}
setShowDeleteConfirm={setShowDeleteConfirm}
handleDeleteClick={handleDeleteClick}
transcriptionError={transcriptionError}
/>
<AudioPlayer
isLoading={isLoading}
error={error}
audioRef={audioRef as React.RefObject<HTMLAudioElement>}
playbackRate={playbackRate}
handlePlaybackRateChange={handlePlaybackRateChange}
handleSeek={handleSeek}
handlePlayPause={handlePlayPause}
containerRef={containerRef as React.RefObject<HTMLDivElement>}
waveformData={waveformData}
currentTime={currentTime}
fileName={fileName}
/>
<audio
ref={audioRef}
src={fileName ? `/api/recordings/${fileName}/recording.wav` : ''}
preload="metadata"
className="hidden"
/>
<TranscriptionStatus
transcription={recording.transcription}
transcriptionError={transcriptionError}
currentAudioTime={currentAudioTime}
/>
<TranscribeButton
transcriptionStatus={recording.transcription?.transcriptionStatus}
onTranscribe={() => void handleTranscribe()}
/>
</div>
);
}
packages/frontend/media-capture-playground/web/components/saved-recordings.tsx
-41
View File
@@ -1,41 +0,0 @@
import useSWRSubscription from 'swr/subscription';
import type { SavedRecording } from '../types';
import { socket } from '../utils';
import { SavedRecordingItem } from './saved-recording-item';
export function SavedRecordings(): React.ReactElement {
const { data: recordings = [] } = useSWRSubscription<SavedRecording[]>(
'saved-recordings',
(
_key: string,
{ next }: { next: (err: Error | null, data?: SavedRecording[]) => void }
) => {
// Subscribe to saved recordings updates
socket.on('apps:saved', (data: { recordings: SavedRecording[] }) => {
next(null, data.recordings);
});
fetch('/api/apps/saved')
.then(res => res.json())
.then(data => next(null, data.recordings))
.catch(err => next(err));
return () => {
socket.off('apps:saved');
};
}
);
if (recordings.length === 0) {
return <p className="text-gray-500 italic text-sm">No saved recordings</p>;
}
return (
<div className="space-y-1">
{recordings.map(recording => (
<SavedRecordingItem key={recording.wav} recording={recording} />
))}
</div>
);
}
packages/frontend/media-capture-playground/web/index.html
-11
View File
@@ -1,11 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Media Capture Playground</title>
</head>
<body>
<div id="root"></div>
</body>
</html>
packages/frontend/media-capture-playground/web/main.css
-1
View File
@@ -1 +0,0 @@
@import 'tailwindcss';
packages/frontend/media-capture-playground/web/main.tsx
-11
View File
@@ -1,11 +0,0 @@
import './main.css';
import { createRoot } from 'react-dom/client';
import { App } from './app';
const rootElement = document.getElementById('root');
if (!rootElement) {
throw new Error('Failed to find the root element');
}
createRoot(rootElement).render(<App />);
packages/frontend/media-capture-playground/web/types.ts
-60
View File
@@ -1,60 +0,0 @@
export interface App {
processId: number;
processGroupId: number;
bundleIdentifier: string;
name: string;
isRunning: boolean;
}
export interface AppGroup {
processGroupId: number;
rootApp: App;
apps: App[];
}
export interface RecordingStatus {
processId: number;
bundleIdentifier: string;
name: string;
startTime: number;
duration: number;
isGlobal?: boolean;
}
export interface RecordingMetadata {
appName: string;
bundleIdentifier: string;
processId: number;
recordingStartTime: number;
recordingEndTime: number;
recordingDuration: number;
sampleRate: number;
channels: number;
totalSamples: number;
icon?: Uint8Array;
mp3: string;
isGlobal?: boolean;
}
export interface TranscriptionMetadata {
transcriptionStartTime: number;
transcriptionEndTime: number;
transcriptionStatus: 'not_started' | 'pending' | 'completed' | 'error';
transcription?: {
title: string;
segments: Array<{
speaker: string;
start_time: string;
end_time: string;
transcription: string;
}>;
summary: string;
};
error?: string;
}
export interface SavedRecording {
wav: string;
metadata?: RecordingMetadata;
transcription?: TranscriptionMetadata;
}
packages/frontend/media-capture-playground/web/utils.ts
-19
View File
@@ -1,19 +0,0 @@
import { io } from 'socket.io-client';
// Create a singleton socket instance
export const socket = io('http://localhost:6544');
export function formatDuration(ms: number): string {
const seconds = Math.floor(ms / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
return `${hours.toString().padStart(2, '0')}:${(minutes % 60)
.toString()
.padStart(2, '0')}:${(seconds % 60).toString().padStart(2, '0')}`;
}
// Helper function to convert timestamp (MM:SS.mmm) to seconds
export function timestampToSeconds(timestamp: string): number {
const [minutes, seconds] = timestamp.split(':').map(parseFloat);
return minutes * 60 + seconds;
}
packages/frontend/native/Cargo.toml
+5 -3
View File
@@ -9,12 +9,14 @@ crate-type = ["cdylib", "rlib"]
[dependencies]
affine_common = { workspace = true, features = ["hashcash"] }
affine_importer = { workspace = true }
affine_media_capture = { path = "./media_capture" }
affine_nbstore = { workspace = true, features = ["napi"] }
affine_sqlite_v1 = { path = "./sqlite_v1" }
napi = { workspace = true }
napi-derive = { workspace = true }
once_cell = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true, default-features = false, features = [
"chrono",
"macros",
@@ -38,9 +40,9 @@ mimalloc = { workspace = true }
mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
[dev-dependencies]
chrono = { workspace = true }
serde_json = { workspace = true }
uuid = { workspace = true }
chrono = { workspace = true }
uuid = { workspace = true }
zip = { workspace = true }
[build-dependencies]
napi-build = { workspace = true }
packages/frontend/native/__tests__/db.spec.mts
+1 -1
View File
@@ -2,7 +2,7 @@ import { fileURLToPath } from 'node:url';
import test from 'ava';
import { SqliteConnection, ValidationResult } from '../index';
import { SqliteConnection, ValidationResult } from '../index.js';
test('db validate', async t => {
const path = fileURLToPath(
packages/frontend/native/__tests__/pool.spec.mts
+1 -1
View File
@@ -1,6 +1,6 @@
import test from 'ava';
import { DocStoragePool } from '../index';
import { DocStoragePool } from '../index.js';
test('can batch read/write pool', async t => {
const pool = new DocStoragePool();
packages/frontend/native/index.d.ts
Vendored
+25 -9
View File
@@ -18,13 +18,6 @@ export declare class ApplicationStateChangedSubscriber {
unsubscribe(): void
}
export declare class AudioCaptureSession {
stop(): void
get sampleRate(): number
get channels(): number
get actualSampleRate(): number
}
export declare class ShareableContent {
static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber
static onAppStateChanged(app: ApplicationInfo, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber
@@ -32,8 +25,6 @@ export declare class ShareableContent {
static applications(): Array<ApplicationInfo>
static applicationWithProcessId(processId: number): ApplicationInfo | null
static isUsingMicrophone(processId: number): boolean
static tapAudio(processId: number, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
static tapGlobalAudio(excludedProcesses: Array<ApplicationInfo> | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
}
export declare function abortRecording(id: string): Promise<void>
@@ -75,6 +66,29 @@ export interface RecordingStartOptions {
export declare function startRecording(opts: RecordingStartOptions): Promise<RecordingSessionMeta>
export declare function stopRecording(id: string): Promise<RecordingArtifact>
export declare function cancelImportSession(sessionId: string): void
export interface CreateImportBatchLimits {
maxDocs?: number
maxBlobs?: number
maxBlobBytes?: number
}
export declare function createImportSession(options: CreateImportSessionOptions): string
export interface CreateImportSessionOptions {
format: string
source: CreateImportSessionSource
batchLimits?: CreateImportBatchLimits
}
export interface CreateImportSessionSource {
kind: string
path: string
}
export declare function disposeImportSession(sessionId: string): void
export interface MermaidRenderOptions {
theme?: string
fontFamily?: string
@@ -92,6 +106,8 @@ export interface MermaidRenderResult {
export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise<string>
export declare function nextImportBatch(sessionId: string): string | null
export declare function renderMermaidSvg(request: MermaidRenderRequest): MermaidRenderResult
export declare function renderTypstSvg(request: TypstRenderRequest): TypstRenderResult
packages/frontend/native/index.js
+4 -1
View File
@@ -575,14 +575,17 @@ module.exports = nativeBinding
module.exports.ApplicationInfo = nativeBinding.ApplicationInfo
module.exports.ApplicationListChangedSubscriber = nativeBinding.ApplicationListChangedSubscriber
module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStateChangedSubscriber
module.exports.AudioCaptureSession = nativeBinding.AudioCaptureSession
module.exports.ShareableContent = nativeBinding.ShareableContent
module.exports.abortRecording = nativeBinding.abortRecording
module.exports.decodeAudio = nativeBinding.decodeAudio
module.exports.decodeAudioSync = nativeBinding.decodeAudioSync
module.exports.startRecording = nativeBinding.startRecording
module.exports.stopRecording = nativeBinding.stopRecording
module.exports.cancelImportSession = nativeBinding.cancelImportSession
module.exports.createImportSession = nativeBinding.createImportSession
module.exports.disposeImportSession = nativeBinding.disposeImportSession
module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse
module.exports.nextImportBatch = nativeBinding.nextImportBatch
module.exports.renderMermaidSvg = nativeBinding.renderMermaidSvg
module.exports.renderTypstSvg = nativeBinding.renderTypstSvg
module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse
packages/frontend/native/media-capture-exapmle.ts
-149
View File
@@ -1,149 +0,0 @@
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
Whisper,
WhisperFullParams,
WhisperSamplingStrategy,
} from '@napi-rs/whisper';
import { BehaviorSubject, EMPTY, Observable } from 'rxjs';
import {
distinctUntilChanged,
exhaustMap,
groupBy,
mergeMap,
switchMap,
tap,
} from 'rxjs/operators';
import { type Application, ShareableContent } from './index.js';
const rootDir = join(fileURLToPath(import.meta.url), '..');
const shareableContent = new ShareableContent();
const appList = new Set([
'com.tinyspeck.slackmacgap.helper',
'us.zoom.xos',
'org.mozilla.firefoxdeveloperedition',
]);
console.info(shareableContent.applications().map(app => app.bundleIdentifier));
const GGLM_LARGE = join(rootDir, 'ggml-large-v3-turbo.bin');
const whisper = new Whisper(GGLM_LARGE, {
useGpu: true,
gpuDevice: 1,
});
const whisperParams = new WhisperFullParams(WhisperSamplingStrategy.Greedy);
const SAMPLE_WINDOW_MS = 3000; // 3 seconds, similar to stream.cpp's step_ms
const SAMPLES_PER_WINDOW = (SAMPLE_WINDOW_MS / 1000) * 16000; // 16kHz sample rate
// eslint-disable-next-line rxjs/finnish
const runningApplications = new BehaviorSubject(
shareableContent.applications()
);
const applicationListChangedSubscriber =
ShareableContent.onApplicationListChanged(() => {
runningApplications.next(shareableContent.applications());
});
runningApplications
.pipe(
mergeMap(apps => apps.filter(app => appList.has(app.bundleIdentifier))),
groupBy(app => app.bundleIdentifier),
mergeMap(app$ =>
app$.pipe(
exhaustMap(app =>
new Observable<[Application, boolean]>(subscriber => {
const stateSubscriber = ShareableContent.onAppStateChanged(
app,
err => {
if (err) {
subscriber.error(err);
return;
}
subscriber.next([app, app.isRunning]);
}
);
return () => {
stateSubscriber.unsubscribe();
};
}).pipe(
distinctUntilChanged(
([_, isRunningA], [__, isRunningB]) => isRunningA === isRunningB
),
switchMap(([app]) =>
!app.isRunning
? EMPTY
: new Observable(observer => {
const buffers: Float32Array[] = [];
const audioStream = app.tapAudio((err, samples) => {
if (err) {
observer.error(err);
return;
}
if (samples) {
buffers.push(samples);
observer.next(samples);
// Calculate total samples in buffer
const totalSamples = buffers.reduce(
(acc, buf) => acc + buf.length,
0
);
// Process when we have enough samples for our window
if (totalSamples >= SAMPLES_PER_WINDOW) {
// Concatenate all buffers
const concatenated = new Float32Array(totalSamples);
let offset = 0;
buffers.forEach(buf => {
concatenated.set(buf, offset);
offset += buf.length;
});
// Transcribe the audio
const result = whisper.full(
whisperParams,
concatenated
);
// Print results
console.info(result);
// Keep any remaining samples for next window
const remainingSamples =
totalSamples - SAMPLES_PER_WINDOW;
if (remainingSamples > 0) {
const lastBuffer = buffers[buffers.length - 1];
buffers.length = 0;
buffers.push(lastBuffer.slice(-remainingSamples));
} else {
buffers.length = 0;
}
}
}
});
return () => {
audioStream.stop();
};
})
)
)
)
)
),
tap({
finalize: () => {
applicationListChangedSubscriber.unsubscribe();
},
})
)
.subscribe();
packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs
-19
View File
@@ -682,14 +682,6 @@ impl ShareableContent {
}
}
#[napi]
pub fn tap_audio(
process_id: u32,
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
ShareableContent::tap_audio_with_callback(process_id, AudioCallback::Js(Arc::new(audio_stream_callback)))
}
pub(crate) fn tap_global_audio_with_callback(
excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: AudioCallback,
@@ -706,15 +698,4 @@ impl ShareableContent {
let boxed_manager = Box::new(device_manager);
Ok(AudioCaptureSession::new(boxed_manager))
}
#[napi]
pub fn tap_global_audio(
excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
ShareableContent::tap_global_audio_with_callback(
excluded_processes,
AudioCallback::Js(Arc::new(audio_stream_callback)),
)
}
}
packages/frontend/native/media_capture/src/macos/tap_audio.rs
-9
View File
@@ -21,7 +21,6 @@ use coreaudio::sys::{
kAudioSubTapUIDKey,
};
use napi::bindgen_prelude::Result;
use napi_derive::napi;
use objc2::runtime::AnyObject;
use crate::{
@@ -887,8 +886,6 @@ impl Drop for AggregateDeviceManager {
}
}
// NEW NAPI Struct: AudioCaptureSession
#[napi]
pub struct AudioCaptureSession {
// Use Option<Box<...>> to allow taking ownership in stop()
manager: Option<Box<AggregateDeviceManager>>,
@@ -896,9 +893,7 @@ pub struct AudioCaptureSession {
channels: Option<u32>,
}
#[napi]
impl AudioCaptureSession {
// Constructor called internally, not directly via NAPI
pub(crate) fn new(manager: Box<AggregateDeviceManager>) -> Self {
Self {
manager: Some(manager),
@@ -907,7 +902,6 @@ impl AudioCaptureSession {
}
}
#[napi]
pub fn stop(&mut self) -> Result<()> {
if let Some(manager) = self.manager.take() {
// Cache the stats before dropping
@@ -926,7 +920,6 @@ impl AudioCaptureSession {
}
}
#[napi(getter)]
pub fn get_sample_rate(&self) -> Result<f64> {
if let Some(manager) = &self.manager {
manager
@@ -943,7 +936,6 @@ impl AudioCaptureSession {
}
}
#[napi(getter)]
pub fn get_channels(&self) -> Result<u32> {
if let Some(manager) = &self.manager {
manager
@@ -960,7 +952,6 @@ impl AudioCaptureSession {
}
}
#[napi(getter)]
pub fn get_actual_sample_rate(&self) -> Result<f64> {
if let Some(manager) = &self.manager {
manager
packages/frontend/native/media_capture/src/windows/audio_capture.rs
-6
View File
@@ -200,7 +200,6 @@ fn emit_mixed_frames(
}
}
#[napi]
pub struct AudioCaptureSession {
mic_stream: Option<cpal::Stream>,
lb_stream: Option<cpal::Stream>,
@@ -258,25 +257,20 @@ where
}
}
#[napi]
impl AudioCaptureSession {
#[napi(getter)]
pub fn get_sample_rate(&self) -> f64 {
self.sample_rate.0 as f64
}
#[napi(getter)]
pub fn get_channels(&self) -> u32 {
self.channels
}
#[napi(getter)]
pub fn get_actual_sample_rate(&self) -> f64 {
// For CPAL we always operate at the target rate which is sample_rate
self.sample_rate.0 as f64
}
#[napi]
pub fn stop(&mut self) -> Result<()> {
teardown_audio_capture_resources(
&mut self.mic_stream,
packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs
-22
View File
@@ -225,16 +225,6 @@ impl ShareableContent {
crate::windows::audio_capture::start_recording(audio_stream_callback, target)
}
#[napi]
pub fn tap_audio(
_process_id: u32, // Currently unused - Windows captures global audio
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
// On Windows with CPAL, we capture global audio (mic + loopback)
// since per-application audio tapping isn't supported the same way as macOS
ShareableContent::tap_audio_with_callback(_process_id, AudioCallback::Js(Arc::new(audio_stream_callback)), None)
}
pub(crate) fn tap_global_audio_with_callback(
_excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: AudioCallback,
@@ -246,18 +236,6 @@ impl ShareableContent {
crate::windows::audio_capture::start_recording(audio_stream_callback, target)
}
#[napi]
pub fn tap_global_audio(
_excluded_processes: Option<Vec<&ApplicationInfo>>,
audio_stream_callback: ThreadsafeFunction<napi::bindgen_prelude::Float32Array, ()>,
) -> Result<AudioCaptureSession> {
ShareableContent::tap_global_audio_with_callback(
_excluded_processes,
AudioCallback::Js(Arc::new(audio_stream_callback)),
None,
)
}
#[napi]
pub fn is_using_microphone(process_id: u32) -> Result<bool> {
is_process_actively_using_microphone(process_id)
packages/frontend/native/nbstore/Cargo.toml
+2 -1
View File
@@ -13,7 +13,8 @@ napi = ["affine_common/napi"]
use-as-lib = ["napi-derive/noop", "napi/noop"]
[dependencies]
affine_common = { workspace = true, features = ["ydoc-loader"] }
affine_common = { workspace = true, features = ["napi"] }
affine_doc_loader = { workspace = true }
affine_schema = { path = "../schema" }
anyhow = { workspace = true }
bincode = { version = "2.0.1", features = ["serde"] }
packages/frontend/native/nbstore/src/error.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use affine_common::doc_parser::ParseError;
use affine_doc_loader::ParseError;
pub type Result<T> = std::result::Result<T, Error>;
packages/frontend/native/nbstore/src/indexer.rs
+2 -2
View File
@@ -1,4 +1,4 @@
use affine_common::doc_parser::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary};
use affine_doc_loader::{BlockInfo, CrawlResult, ParseError, parse_doc_from_binary};
use memory_indexer::{SearchHit, SnapshotData};
use napi_derive::napi;
use serde::Serialize;
@@ -276,7 +276,7 @@ fn merge_updates(mut segments: Vec<Vec<u8>>, guid: &str) -> Result<Vec<u8>> {
mod tests {
use std::path::{Path, PathBuf};
use affine_common::doc_parser::ParseError;
use affine_doc_loader::ParseError;
use chrono::Utc;
use serde_json::Value;
use tokio::fs;
packages/frontend/native/nbstore/src/storage.rs
+1 -1
View File
@@ -311,7 +311,7 @@ mod tests {
storage
.set_blob(crate::SetBlob {
key: "large-blob".to_string(),
data: vec![7; 1024 * 1024],
data: Into::<crate::Data>::into(vec![7; 1024 * 1024]),
mime: "application/octet-stream".to_string(),
})
.await
packages/frontend/native/src/import_session.rs
+270
View File
@@ -0,0 +1,270 @@
use std::{
collections::HashMap,
path::PathBuf,
sync::{
Mutex,
atomic::{AtomicU64, Ordering},
},
};
use affine_importer::{
ImportBatchLimits, ImportFormat, ImportOptions, ImportSession, ImportSessionOptions, ImportSessionSource,
};
use napi::{Status, bindgen_prelude::*};
use napi_derive::napi;
use once_cell::sync::Lazy;
static IMPORT_SESSIONS: Lazy<Mutex<HashMap<String, ImportSession>>> = Lazy::new(|| Mutex::new(HashMap::new()));
static NEXT_SESSION_ID: AtomicU64 = AtomicU64::new(1);
#[napi(object)]
pub struct CreateImportSessionOptions {
pub format: String,
pub source: CreateImportSessionSource,
pub batch_limits: Option<CreateImportBatchLimits>,
}
#[napi(object)]
pub struct CreateImportSessionSource {
pub kind: String,
pub path: String,
}
#[napi(object)]
pub struct CreateImportBatchLimits {
pub max_docs: Option<u32>,
pub max_blobs: Option<u32>,
pub max_blob_bytes: Option<i64>,
}
fn parse_format(format: &str) -> Result<ImportFormat> {
match format {
"markdownZip" => Ok(ImportFormat::MarkdownZip),
"notionZip" => Ok(ImportFormat::NotionZip),
"notionMarkdownZip" => Ok(ImportFormat::NotionMarkdownZip),
"notionHtmlZip" => Ok(ImportFormat::NotionHtmlZip),
"obsidian" => Ok(ImportFormat::Obsidian),
"bearZip" => Ok(ImportFormat::BearZip),
_ => Err(Error::new(
Status::InvalidArg,
format!("unsupported import format: {format}"),
)),
}
}
fn map_import_error(error: affine_importer::ImportError) -> Error {
let status = match error {
affine_importer::ImportError::Cancelled => Status::Cancelled,
affine_importer::ImportError::UnsupportedFormat | affine_importer::ImportError::InvalidSource(_) => {
Status::InvalidArg
}
affine_importer::ImportError::Zip(_)
| affine_importer::ImportError::Io(_)
| affine_importer::ImportError::Document(_) => Status::GenericFailure,
};
Error::new(status, error.to_string())
}
#[napi]
pub fn create_import_session(options: CreateImportSessionOptions) -> Result<String> {
let format = parse_format(&options.format)?;
let source = match options.source.kind.as_str() {
"filePath" => ImportSessionSource::FilePath(PathBuf::from(options.source.path)),
"directoryPath" => ImportSessionSource::DirectoryPath(PathBuf::from(options.source.path)),
kind => {
return Err(Error::new(
Status::InvalidArg,
format!("unsupported import source kind: {kind}"),
));
}
};
let mut batch_limits = ImportBatchLimits::default();
if let Some(limits) = options.batch_limits {
if let Some(max_docs) = limits.max_docs {
batch_limits.max_docs = max_docs as usize;
}
if let Some(max_blobs) = limits.max_blobs {
batch_limits.max_blobs = max_blobs as usize;
}
if let Some(max_blob_bytes) = limits.max_blob_bytes {
if !(0..=100 * 1024 * 1024).contains(&max_blob_bytes) {
return Err(Error::new(Status::InvalidArg, "maxBlobBytes not valid"));
}
batch_limits.max_blob_bytes = max_blob_bytes as u64;
}
}
let session = ImportSession::create(ImportSessionOptions {
format,
source,
import_options: ImportOptions::default(),
batch_limits,
})
.map_err(map_import_error)?;
let id = format!("import-session-{}", NEXT_SESSION_ID.fetch_add(1, Ordering::Relaxed));
IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?
.insert(id.clone(), session);
Ok(id)
}
#[napi]
pub fn next_import_batch(session_id: String) -> Result<Option<String>> {
let mut sessions = IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?;
let session = sessions
.get_mut(&session_id)
.ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?;
session
.next_batch()
.map_err(map_import_error)?
.map(|batch| serde_json::to_string(&batch).map_err(|err| Error::new(Status::GenericFailure, err.to_string())))
.transpose()
}
#[napi]
pub fn cancel_import_session(session_id: String) -> Result<()> {
let mut sessions = IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?;
let session = sessions
.get_mut(&session_id)
.ok_or_else(|| Error::new(Status::InvalidArg, format!("unknown import session: {session_id}")))?;
session.cancel();
Ok(())
}
#[napi]
pub fn dispose_import_session(session_id: String) -> Result<()> {
IMPORT_SESSIONS
.lock()
.map_err(|_| Error::new(Status::GenericFailure, "import session registry poisoned"))?
.remove(&session_id);
Ok(())
}
#[cfg(test)]
mod tests {
use std::{
fs,
io::Write,
path::PathBuf,
sync::atomic::{AtomicU64, Ordering},
};
use zip::{ZipWriter, write::SimpleFileOptions};
use super::*;
static NEXT_TEST_ARCHIVE_ID: AtomicU64 = AtomicU64::new(1);
fn archive_path(entries: &[(&str, &[u8])]) -> PathBuf {
let path = std::env::temp_dir().join(format!(
"affine-native-import-{}-{}.zip",
std::process::id(),
NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed)
));
let file = std::fs::File::create(&path).unwrap();
let mut writer = ZipWriter::new(file);
for (path, bytes) in entries {
writer.start_file(path, SimpleFileOptions::default()).unwrap();
writer.write_all(bytes).unwrap();
}
writer.finish().unwrap();
path
}
fn directory_path(entries: &[(&str, &[u8])]) -> PathBuf {
let root = std::env::temp_dir().join(format!(
"affine-native-import-dir-{}-{}",
std::process::id(),
NEXT_TEST_ARCHIVE_ID.fetch_add(1, Ordering::Relaxed)
));
for (path, bytes) in entries {
let path = root.join(path);
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, bytes).unwrap();
}
root
}
#[test]
fn import_session_returns_one_serialized_batch() {
let path = archive_path(&[("entry.md", b"entry")]);
let id = create_import_session(CreateImportSessionOptions {
format: "markdownZip".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
})
.unwrap();
let batch = next_import_batch(id.clone()).unwrap().unwrap();
assert!(batch.contains("\"docs\""));
assert!(next_import_batch(id.clone()).unwrap().is_none());
dispose_import_session(id).unwrap();
let _ = std::fs::remove_file(path);
}
#[test]
fn import_session_cancel_blocks_next_batch() {
let path = archive_path(&[("entry.md", b"entry")]);
let id = create_import_session(CreateImportSessionOptions {
format: "markdownZip".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
})
.unwrap();
cancel_import_session(id.clone()).unwrap();
assert!(next_import_batch(id.clone()).is_err());
dispose_import_session(id).unwrap();
let _ = std::fs::remove_file(path);
}
#[test]
fn import_session_accepts_directory_source() {
let path = directory_path(&[("entry.md", b"entry")]);
let id = create_import_session(CreateImportSessionOptions {
format: "obsidian".to_string(),
source: CreateImportSessionSource {
kind: "directoryPath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: None,
})
.unwrap();
let batch = next_import_batch(id.clone()).unwrap().unwrap();
assert!(batch.contains("\"docs\""));
assert!(next_import_batch(id.clone()).unwrap().is_none());
dispose_import_session(id).unwrap();
let _ = fs::remove_dir_all(path);
}
#[test]
fn import_session_rejects_negative_blob_byte_limit() {
let path = archive_path(&[("entry.md", b"entry")]);
let result = create_import_session(CreateImportSessionOptions {
format: "markdownZip".to_string(),
source: CreateImportSessionSource {
kind: "filePath".to_string(),
path: path.to_string_lossy().to_string(),
},
batch_limits: Some(CreateImportBatchLimits {
max_docs: None,
max_blobs: None,
max_blob_bytes: Some(-1),
}),
});
assert!(result.is_err());
let _ = std::fs::remove_file(path);
}
}
packages/frontend/native/src/lib.rs
+2
View File
@@ -1,4 +1,5 @@
pub mod hashcash;
mod import_session;
#[cfg(not(any(target_os = "android", target_os = "ios")))]
pub mod preview;
@@ -10,3 +11,4 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
pub use affine_media_capture::*;
pub use affine_nbstore::*;
pub use affine_sqlite_v1::*;
pub use import_session::*;
tools/cli/src/bundle-shared.ts
-1
View File
@@ -9,7 +9,6 @@ export const RSPACK_SUPPORTED_PACKAGES = [
'@affine/electron-renderer',
'@affine/server',
'@affine/reader',
'@affine/media-capture-playground',
] as const;
const rspackSupportedPackageSet = new Set<string>(RSPACK_SUPPORTED_PACKAGES);

Some files were not shown because too many files have changed in this diff Show More