mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-03-24 16:18:39 +08:00
fix #14592 ### Description > 🤖 **Note:** The code in this Pull Request were developed with the assistance of AI, but have been thoroughly reviewed and manually tested. > I noticed there's a check when opening an issue that asks _"Is your content generated by AI?"_, so I mention it here in case it's a deal breaker. If so I understand, you can close the PR, just wanted to share this in case it's useful anyways. This PR introduces **Obsidian Vault Import Support** to AFFiNE. Previously, users migrating from Obsidian had to rely on the generic Markdown importer, which often resulted in broken cross-links, missing directory structures, and metadata conflicts because Obsidian relies heavily on proprietary structures not supported by standard Markdown. This completely new feature makes migrating to AFFiNE easy. **Key Features & Implementations:** 1. **Vault (Directory) Selection** - Utilizes the `openDirectory` blocksuite utility in the import modal to allow users to select an entire folder directly from their filesystem, maintaining file context rather than forcing `.zip` uploads. 2. **Wikilink Resolution (Two-Pass Import)** - Restructured the `importObsidianVault` process into a two-pass architecture. - **Pass 1:** Discovers all files, assigns new AFFiNE document IDs, and maps them efficiently (by title, alias, and filename) into a high-performance hash map. - **Pass 2:** Processes the generic markdown AST and correctly maps custom `[[wikilinks]]` to the actual pre-registered AFFiNE blocksuite document IDs via `obsidianWikilinkToDeltaMatcher`. - Safely strips leading emojis from wikilink aliases to prevent duplicated page icons rendering mid-sentence. 3. **Emoji Metadata & State Fixes** - Implemented an aggressive, single-pass RegExp to extract multiple leading/combining emojis (`Emoji_Presentation` / `\ufe0f`) from H1 headers and Frontmatter. Emojis are assigned specifically to the page icon metadata property and cleanly stripped from the visual document title. - Fixed a core mutation bug where the loop iterating over existing `docMetas` was aggressively overwriting newly minted IDs for the current import batch. This fully resolves the issue where imported pages (especially re-imports) were incorrectly flagged as `trashed`. - Enforces explicit `trash: false` patch instructions. 4. **Syntax Conversion** - Implemented conversion of Obsidian-style Callouts (`> [!NOTE] Title`) into native AFFiNE block formats (`> 💡 **Title**`). - Hardened the `blockquote` parser so that nested structures (like `> - list items`) are fully preserved instead of discarded. ### UI Changes - Updated the Import Modal to include the "Import Obsidian Vault" flow utilizing the native filesystem directory picker. - Regenerated and synced `i18n-completenesses.json` correctly up to 100% across all supported locales for the new modal string additions. ### Testing Instructions 1. Navigate to the Workspace sidebar and click "Import". 2. Select "Obsidian" and use the directory picker to define a comprehensive Vault folder. 3. Validate that cross-links between documents automatically resolve to their specific AFFiNE instances. 4. Validate documents containing leading Emojis display exactly one Emoji (in the page icon area), and none duplicated in the actual title header. 5. Validate Callouts are rendered cleanly and correctly, and no documents are incorrectly marked as "Trash". <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Import Obsidian vaults with wikilink resolution, emoji/title preservation, asset handling, and automatic document creation. * Folder-based imports via a Directory Picker (with hidden-input fallback) integrated into the import dialog. * **Localization** * Added Obsidian import label and tooltip translations. * **Tests** * Added end-to-end tests validating Obsidian vault import and asset handling. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: DarkSky <25152247+darkskygit@users.noreply.github.com> Co-authored-by: DarkSky <darksky2048@gmail.com>
398 lines
10 KiB
TypeScript
398 lines
10 KiB
TypeScript
// Polyfill for `showOpenFilePicker` API
|
|
// See https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/wicg-file-system-access/index.d.ts
|
|
// See also https://caniuse.com/?search=showOpenFilePicker
|
|
import { BlockSuiteError, ErrorCode } from '@blocksuite/global/exceptions';
|
|
|
|
interface OpenFilePickerOptions {
|
|
types?:
|
|
| {
|
|
description?: string | undefined;
|
|
accept: Record<string, string | string[]>;
|
|
}[]
|
|
| undefined;
|
|
excludeAcceptAllOption?: boolean | undefined;
|
|
multiple?: boolean | undefined;
|
|
}
|
|
|
|
declare global {
|
|
interface Window {
|
|
// Window API: showOpenFilePicker
|
|
showOpenFilePicker?: (
|
|
options?: OpenFilePickerOptions
|
|
) => Promise<FileSystemFileHandle[]>;
|
|
// Window API: showDirectoryPicker
|
|
showDirectoryPicker?: (options?: {
|
|
id?: string;
|
|
mode?: 'read' | 'readwrite';
|
|
startIn?: FileSystemHandle | string;
|
|
}) => Promise<FileSystemDirectoryHandle>;
|
|
}
|
|
}
|
|
|
|
// Minimal polyfill for FileSystemDirectoryHandle to iterate over files
|
|
interface FileSystemDirectoryHandle {
|
|
kind: 'directory';
|
|
name: string;
|
|
values(): AsyncIterableIterator<
|
|
FileSystemFileHandle | FileSystemDirectoryHandle
|
|
>;
|
|
}
|
|
|
|
interface FileSystemFileHandle {
|
|
kind: 'file';
|
|
name: string;
|
|
getFile(): Promise<File>;
|
|
}
|
|
|
|
// See [Common MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types)
|
|
const FileTypes: NonNullable<OpenFilePickerOptions['types']> = [
|
|
{
|
|
description: 'Images',
|
|
accept: {
|
|
'image/*': [
|
|
'.avif',
|
|
'.gif',
|
|
// '.ico',
|
|
'.jpeg',
|
|
'.jpg',
|
|
'.png',
|
|
'.tif',
|
|
'.tiff',
|
|
// '.svg',
|
|
'.webp',
|
|
],
|
|
},
|
|
},
|
|
{
|
|
description: 'Videos',
|
|
accept: {
|
|
'video/*': [
|
|
'.avi',
|
|
'.mp4',
|
|
'.mpeg',
|
|
'.ogg',
|
|
// '.ts',
|
|
'.webm',
|
|
'.3gp',
|
|
'.3g2',
|
|
],
|
|
},
|
|
},
|
|
{
|
|
description: 'Audios',
|
|
accept: {
|
|
'audio/*': [
|
|
'.aac',
|
|
'.mid',
|
|
'.midi',
|
|
'.mp3',
|
|
'.oga',
|
|
'.opus',
|
|
'.wav',
|
|
'.weba',
|
|
'.3gp',
|
|
'.3g2',
|
|
],
|
|
},
|
|
},
|
|
{
|
|
description: 'Markdown',
|
|
accept: {
|
|
'text/markdown': ['.md', '.markdown'],
|
|
},
|
|
},
|
|
{
|
|
description: 'Html',
|
|
accept: {
|
|
'text/html': ['.html', '.htm'],
|
|
},
|
|
},
|
|
{
|
|
description: 'Zip',
|
|
accept: {
|
|
'application/zip': ['.zip'],
|
|
},
|
|
},
|
|
{
|
|
description: 'Docx',
|
|
accept: {
|
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
|
|
['.docx'],
|
|
},
|
|
},
|
|
{
|
|
description: 'MindMap',
|
|
accept: {
|
|
'text/xml': ['.mm', '.opml', '.xml'],
|
|
},
|
|
},
|
|
];
|
|
|
|
/**
|
|
* See https://web.dev/patterns/files/open-one-or-multiple-files/
|
|
*/
|
|
type AcceptTypes =
|
|
| 'Any'
|
|
| 'Images'
|
|
| 'Videos'
|
|
| 'Audios'
|
|
| 'Markdown'
|
|
| 'Html'
|
|
| 'Zip'
|
|
| 'Docx'
|
|
| 'MindMap';
|
|
|
|
function canUseFileSystemAccessAPI(
|
|
api: 'showOpenFilePicker' | 'showDirectoryPicker'
|
|
) {
|
|
return (
|
|
api in window &&
|
|
(() => {
|
|
try {
|
|
return window.self === window.top;
|
|
} catch {
|
|
return false;
|
|
}
|
|
})()
|
|
);
|
|
}
|
|
|
|
export async function openFilesWith(
|
|
acceptType: AcceptTypes = 'Any',
|
|
multiple: boolean = true
|
|
): Promise<File[] | null> {
|
|
const supportsFileSystemAccess =
|
|
canUseFileSystemAccessAPI('showOpenFilePicker');
|
|
|
|
// If the File System Access API is supported…
|
|
if (supportsFileSystemAccess && window.showOpenFilePicker) {
|
|
try {
|
|
const fileType = FileTypes.find(i => i.description === acceptType);
|
|
if (acceptType !== 'Any' && !fileType)
|
|
throw new BlockSuiteError(
|
|
ErrorCode.DefaultRuntimeError,
|
|
`Unexpected acceptType "${acceptType}"`
|
|
);
|
|
const pickerOpts = {
|
|
types: fileType ? [fileType] : undefined,
|
|
multiple,
|
|
} satisfies OpenFilePickerOptions;
|
|
// Show the file picker, optionally allowing multiple files.
|
|
const handles = await window.showOpenFilePicker(pickerOpts);
|
|
|
|
return await Promise.all(handles.map(handle => handle.getFile()));
|
|
} catch (err) {
|
|
console.error(err);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Fallback if the File System Access API is not supported.
|
|
return new Promise(resolve => {
|
|
// Append a new `<input type="file" multiple? />` and hide it.
|
|
const input = document.createElement('input');
|
|
input.classList.add('affine-upload-input');
|
|
input.style.display = 'none';
|
|
input.type = 'file';
|
|
input.multiple = multiple;
|
|
|
|
if (acceptType !== 'Any') {
|
|
// For example, `accept="image/*"` or `accept="video/*,audio/*"`.
|
|
input.accept = Object.keys(
|
|
FileTypes.find(i => i.description === acceptType)?.accept ?? ''
|
|
).join(',');
|
|
}
|
|
document.body.append(input);
|
|
// The `change` event fires when the user interacts with the dialog.
|
|
input.addEventListener('change', () => {
|
|
// Remove the `<input type="file" multiple? />` again from the DOM.
|
|
input.remove();
|
|
|
|
resolve(input.files ? Array.from(input.files) : null);
|
|
});
|
|
// The `cancel` event fires when the user cancels the dialog.
|
|
input.addEventListener('cancel', () => resolve(null));
|
|
// Show the picker.
|
|
if ('showPicker' in HTMLInputElement.prototype) {
|
|
input.showPicker();
|
|
} else {
|
|
input.click();
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function openDirectory(): Promise<File[] | null> {
|
|
const supportsFileSystemAccess = canUseFileSystemAccessAPI(
|
|
'showDirectoryPicker'
|
|
);
|
|
|
|
if (supportsFileSystemAccess && window.showDirectoryPicker) {
|
|
try {
|
|
const dirHandle = await window.showDirectoryPicker();
|
|
const files: File[] = [];
|
|
|
|
const readDirectory = async (
|
|
directoryHandle: FileSystemDirectoryHandle,
|
|
path: string
|
|
) => {
|
|
for await (const handle of directoryHandle.values()) {
|
|
const relativePath = path ? `${path}/${handle.name}` : handle.name;
|
|
if (handle.kind === 'file') {
|
|
const fileHandle = handle as FileSystemFileHandle;
|
|
if (fileHandle.getFile) {
|
|
const file = await fileHandle.getFile();
|
|
Object.defineProperty(file, 'webkitRelativePath', {
|
|
value: relativePath,
|
|
writable: false,
|
|
});
|
|
files.push(file);
|
|
}
|
|
} else if (handle.kind === 'directory') {
|
|
await readDirectory(
|
|
handle as FileSystemDirectoryHandle,
|
|
relativePath
|
|
);
|
|
}
|
|
}
|
|
};
|
|
|
|
await readDirectory(dirHandle, '');
|
|
return files;
|
|
} catch (err) {
|
|
console.error(err);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
return new Promise(resolve => {
|
|
const input = document.createElement('input');
|
|
input.classList.add('affine-upload-input');
|
|
input.style.display = 'none';
|
|
input.type = 'file';
|
|
|
|
input.setAttribute('webkitdirectory', '');
|
|
input.setAttribute('directory', '');
|
|
|
|
document.body.append(input);
|
|
|
|
input.addEventListener('change', () => {
|
|
input.remove();
|
|
resolve(input.files ? Array.from(input.files) : null);
|
|
});
|
|
|
|
input.addEventListener('cancel', () => resolve(null));
|
|
|
|
if ('showPicker' in HTMLInputElement.prototype) {
|
|
input.showPicker();
|
|
} else {
|
|
input.click();
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function openSingleFileWith(
|
|
acceptType?: AcceptTypes
|
|
): Promise<File | null> {
|
|
const files = await openFilesWith(acceptType, false);
|
|
return files?.at(0) ?? null;
|
|
}
|
|
|
|
export async function getImageFilesFromLocal() {
|
|
const files = await openFilesWith('Images');
|
|
return files ?? [];
|
|
}
|
|
|
|
export function downloadBlob(blob: Blob, name: string) {
|
|
const dataURL = URL.createObjectURL(blob);
|
|
const tmpLink = document.createElement('a');
|
|
const event = new MouseEvent('click');
|
|
tmpLink.download = name;
|
|
tmpLink.href = dataURL;
|
|
tmpLink.dispatchEvent(event);
|
|
|
|
tmpLink.remove();
|
|
URL.revokeObjectURL(dataURL);
|
|
}
|
|
|
|
// Use lru strategy is a better choice, but it's just a temporary solution.
|
|
const MAX_TEMP_DATA_SIZE = 100;
|
|
/**
|
|
* TODO @Saul-Mirone use some other way to store the temp data
|
|
*
|
|
* @deprecated Waiting for migration
|
|
*/
|
|
const tempAttachmentMap = new Map<
|
|
string,
|
|
{
|
|
// name for the attachment
|
|
name: string;
|
|
}
|
|
>();
|
|
const tempImageMap = new Map<
|
|
string,
|
|
{
|
|
// This information comes from pictures.
|
|
// If the user switches between pictures and attachments,
|
|
// this information should be retained.
|
|
width: number | undefined;
|
|
height: number | undefined;
|
|
}
|
|
>();
|
|
|
|
/**
|
|
* Because the image block and attachment block have different props.
|
|
* We need to save some data temporarily when converting between them to ensure no data is lost.
|
|
*
|
|
* For example, before converting from an image block to an attachment block,
|
|
* we need to save the image's width and height.
|
|
*
|
|
* Similarly, when converting from an attachment block to an image block,
|
|
* we need to save the attachment's name.
|
|
*
|
|
* See also https://github.com/toeverything/blocksuite/pull/4583#pullrequestreview-1610662677
|
|
*
|
|
* @internal
|
|
*/
|
|
export function withTempBlobData() {
|
|
const saveAttachmentData = (sourceId: string, data: { name: string }) => {
|
|
if (tempAttachmentMap.size > MAX_TEMP_DATA_SIZE) {
|
|
console.warn(
|
|
'Clear the temp attachment data. It may cause filename loss when converting between image and attachment.'
|
|
);
|
|
tempAttachmentMap.clear();
|
|
}
|
|
|
|
tempAttachmentMap.set(sourceId, data);
|
|
};
|
|
const getAttachmentData = (blockId: string) => {
|
|
const data = tempAttachmentMap.get(blockId);
|
|
tempAttachmentMap.delete(blockId);
|
|
return data;
|
|
};
|
|
|
|
const saveImageData = (
|
|
sourceId: string,
|
|
data: { width: number | undefined; height: number | undefined }
|
|
) => {
|
|
if (tempImageMap.size > MAX_TEMP_DATA_SIZE) {
|
|
console.warn(
|
|
'Clear temp image data. It may cause image width and height loss when converting between image and attachment.'
|
|
);
|
|
tempImageMap.clear();
|
|
}
|
|
|
|
tempImageMap.set(sourceId, data);
|
|
};
|
|
const getImageData = (blockId: string) => {
|
|
const data = tempImageMap.get(blockId);
|
|
tempImageMap.delete(blockId);
|
|
return data;
|
|
};
|
|
return {
|
|
saveAttachmentData,
|
|
getAttachmentData,
|
|
saveImageData,
|
|
getImageData,
|
|
};
|
|
}
|