Jayden/AFFiNE-Mirror
1
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-02-04 00:28:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(editor): support frontmatter & colored text parsing (#14205)

Browse Source 
fix #13847
This commit is contained in:
DarkSky
2026-01-03 22:43:11 +08:00
committed by GitHub
parent 510933becf
commit fe5d6c0c0f
16 changed files with 1033 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

660
.yarn/releases/yarn-4.9.1.cjs → .yarn/releases/yarn-4.12.0.cjs vendored
View File

File diff suppressed because one or more lines are too long

2
.yarnrc.yml
View File

@@ -12,4 +12,4 @@ npmPublishAccess: public
npmRegistryServer: "https://registry.npmjs.org"
yarnPath: .yarn/releases/yarn-4.9.1.cjs
yarnPath: .yarn/releases/yarn-4.12.0.cjs

92
Cargo.lock generated
View File

@@ -669,7 +669,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
dependencies = [
"memchr",
"regex-automata 0.4.9",
"regex-automata",
"serde",
]
@@ -1490,7 +1490,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -1557,8 +1557,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
dependencies = [
"bit-set 0.5.3",
"regex-automata 0.4.9",
"regex-syntax 0.8.5",
"regex-automata",
"regex-syntax",
]
[[package]]
@@ -2012,7 +2012,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.61.2",
"windows-core 0.57.0",
]
[[package]]
@@ -2263,7 +2263,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -2457,7 +2457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
dependencies = [
"cfg-if",
"windows-targets 0.52.6",
"windows-targets 0.48.5",
]
[[package]]
@@ -2618,11 +2618,11 @@ dependencies = [
[[package]]
name = "matchers"
version = "0.1.0"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
dependencies = [
"regex-automata 0.1.10",
"regex-automata",
]
[[package]]
@@ -2861,12 +2861,11 @@ dependencies = [
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"overload",
"winapi",
"windows-sys 0.59.0",
]
[[package]]
@@ -3082,12 +3081,6 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "parking"
version = "2.2.1"
@@ -3471,7 +3464,7 @@ dependencies = [
"rand 0.8.5",
"rand_chacha 0.3.1",
"rand_xorshift",
"regex-syntax 0.8.5",
"regex-syntax",
"rusty-fork",
"tempfile",
"unarray",
@@ -3663,17 +3656,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.4.9",
"regex-syntax 0.8.5",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax 0.6.29",
"regex-automata",
"regex-syntax",
]
[[package]]
@@ -3684,15 +3668,9 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.5",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.5"
@@ -3837,7 +3815,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -4686,7 +4664,7 @@ dependencies = [
"getrandom 0.3.3",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -4940,14 +4918,14 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
version = "0.3.19"
version = "0.3.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
dependencies = [
"matchers",
"nu-ansi-term",
"once_cell",
"regex",
"regex-automata",
"sharded-slab",
"smallvec",
"thread_local",
@@ -4974,7 +4952,7 @@ checksum = "6d7b8994f367f16e6fa14b5aebbcb350de5d7cbea82dc5b00ae997dd71680dd2"
dependencies = [
"cc",
"regex",
"regex-syntax 0.8.5",
"regex-syntax",
"serde_json",
"streaming-iterator",
"tree-sitter-language",
@@ -5560,37 +5538,15 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.48.0",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.54.0"

83
blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts
View File

@@ -1,3 +1,4 @@
import { MarkdownTransformer } from '@blocksuite/affine/widgets/linked-doc';
import {
DefaultTheme,
NoteDisplayMode,
@@ -16,12 +17,15 @@ import type {
SliceSnapshot,
TransformerMiddleware,
} from '@blocksuite/store';
import { AssetsManager, MemoryBlobCRUD } from '@blocksuite/store';
import { AssetsManager, MemoryBlobCRUD, Schema } from '@blocksuite/store';
import { TestWorkspace } from '@blocksuite/store/test';
import { describe, expect, test } from 'vitest';
import { AffineSchemas } from '../../schemas.js';
import { createJob } from '../utils/create-job.js';
import { getProvider } from '../utils/get-provider.js';
import { nanoidReplacement } from '../utils/nanoid-replacement.js';
import { testStoreExtensions } from '../utils/store.js';
const provider = getProvider();
@@ -90,6 +94,39 @@ describe('snapshot to markdown', () => {
expect(target.file).toBe(markdown);
});
test('imports frontmatter metadata into doc meta', async () => {
const schema = new Schema().register(AffineSchemas);
const collection = new TestWorkspace();
collection.storeExtensions = testStoreExtensions;
collection.meta.initialize();
const markdown = `---
title: Web developer
created: 2018-04-12T09:51:00
updated: 2018-04-12T10:00:00
tags: [a, b]
favorite: true
---
Hello world
`;
const docId = await MarkdownTransformer.importMarkdownToDoc({
collection,
schema,
markdown,
fileName: 'fallback-title',
extensions: testStoreExtensions,
});
expect(docId).toBeTruthy();
const meta = collection.meta.getDocMeta(docId!);
expect(meta?.title).toBe('Web developer');
expect(meta?.createDate).toBe(Date.parse('2018-04-12T09:51:00'));
expect(meta?.updatedDate).toBe(Date.parse('2018-04-12T10:00:00'));
expect(meta?.favorite).toBe(true);
expect(meta?.tags).toEqual(['a', 'b']);
});
test('paragraph', async () => {
const blockSnapshot: BlockSnapshot = {
type: 'block',
@@ -2996,6 +3033,50 @@ describe('markdown to snapshot', () => {
});
});
test('html inline color span imports to nearest supported text color', async () => {
const markdown = `<span style="color: #00afde;">Hello</span>`;
const blockSnapshot: BlockSnapshot = {
type: 'block',
id: 'matchesReplaceMap[0]',
flavour: 'affine:note',
props: {
xywh: '[0,0,800,95]',
background: DefaultTheme.noteBackgrounColor,
index: 'a0',
hidden: false,
displayMode: NoteDisplayMode.DocAndEdgeless,
},
children: [
{
type: 'block',
id: 'matchesReplaceMap[1]',
flavour: 'affine:paragraph',
props: {
type: 'text',
text: {
'$blocksuite:internal:text$': true,
delta: [
{
insert: 'Hello',
attributes: {
color: 'var(--affine-v2-text-highlight-fg-blue)',
},
},
],
},
},
children: [],
},
],
};
const mdAdapter = new MarkdownAdapter(createJob(), provider);
const rawBlockSnapshot = await mdAdapter.toBlockSnapshot({
file: markdown,
});
expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot);
});
test('paragraph', async () => {
const markdown = `aaa

120
blocksuite/affine/inlines/preset/src/adapters/html/color-utils.ts Normal file
View File

@@ -0,0 +1,120 @@
import { parseStringToRgba } from '@blocksuite/affine-components/color-picker';
import { cssVarV2, darkThemeV2, lightThemeV2 } from '@toeverything/theme/v2';
type Rgb = { r: number; g: number; b: number };
const COLOR_DISTANCE_THRESHOLD = 90;
const supportedTextColorNames = [
'red',
'orange',
'yellow',
'green',
'teal',
'blue',
'purple',
'grey',
] as const;
const supportedTextColors = supportedTextColorNames.map(name => ({
name,
cssVar: cssVarV2(`text/highlight/fg/${name}`),
light: lightThemeV2[`text/highlight/fg/${name}`],
dark: darkThemeV2[`text/highlight/fg/${name}`],
}));
const hexToRgb = (value: string): Rgb | null => {
const hex = value.replace('#', '');
if (![3, 4, 6, 8].includes(hex.length)) {
return null;
}
const normalized =
hex.length === 3 || hex.length === 4
? hex
.slice(0, 3)
.split('')
.map(c => c + c)
.join('')
: hex.slice(0, 6);
const intVal = Number.parseInt(normalized, 16);
if (Number.isNaN(intVal)) {
return null;
}
return {
r: (intVal >> 16) & 255,
g: (intVal >> 8) & 255,
b: intVal & 255,
};
};
export const parseCssColor = (value: string): Rgb | null => {
const trimmed = value.trim();
if (!trimmed) {
return null;
}
if (trimmed.startsWith('#')) {
return hexToRgb(trimmed);
}
if (/^rgba?\(/i.test(trimmed)) {
const rgba = parseStringToRgba(trimmed);
return {
r: Math.round(rgba.r * 255),
g: Math.round(rgba.g * 255),
b: Math.round(rgba.b * 255),
};
}
return null;
};
const colorDistance = (a: Rgb, b: Rgb) => {
const dr = a.r - b.r;
const dg = a.g - b.g;
const db = a.b - b.b;
return Math.sqrt(dr * dr + dg * dg + db * db);
};
export const resolveNearestSupportedColor = (color: string): string | null => {
const target = parseCssColor(color);
if (!target) {
return null;
}
let nearest:
| {
cssVar: string;
distance: number;
}
| undefined;
for (const supported of supportedTextColors) {
const light = parseCssColor(supported.light);
const dark = parseCssColor(supported.dark);
for (const ref of [light, dark]) {
if (!ref) continue;
const distance = colorDistance(target, ref);
if (!nearest || distance < nearest.distance) {
nearest = { cssVar: supported.cssVar, distance };
}
}
}
if (nearest && nearest.distance <= COLOR_DISTANCE_THRESHOLD) {
return nearest.cssVar;
}
return null;
};
export const extractColorFromStyle = (
style: string | undefined
): string | null => {
if (typeof style !== 'string') {
return null;
}
const declarations = style.split(';');
for (const declaration of declarations) {
const [rawKey, rawValue] = declaration.split(':');
if (!rawKey || !rawValue) continue;
if (rawKey.trim().toLowerCase() === 'color') {
return rawValue.trim();
}
}
return null;
};

40
blocksuite/affine/inlines/preset/src/adapters/html/html-inline.ts
View File

@@ -5,6 +5,11 @@ import {
import { collapseWhiteSpace } from 'collapse-white-space';
import type { Element } from 'hast';
import {
extractColorFromStyle,
resolveNearestSupportedColor,
} from './color-utils.js';
/**
* Handle empty text nodes created by HTML parser for styling purposes.
* These nodes typically contain only whitespace/newlines, for example:
@@ -173,6 +178,40 @@ export const htmlTextToDeltaMatcher = HtmlASTToDeltaExtension({
},
});
export const htmlColorStyleElementToDeltaMatcher = HtmlASTToDeltaExtension({
name: 'color-style-element',
match: ast =>
isElement(ast) &&
ast.tagName === 'span' &&
typeof ast.properties?.style === 'string' &&
/color\s*:/i.test(ast.properties.style),
toDelta: (ast, context) => {
if (!isElement(ast)) {
return [];
}
const baseOptions = { ...context.options, trim: false };
// In preformatted contexts (e.g. code blocks) we don't keep inline colors.
if (baseOptions.pre) {
return ast.children.flatMap(child => context.toDelta(child, baseOptions));
}
const colorValue = extractColorFromStyle(
typeof ast.properties?.style === 'string' ? ast.properties.style : ''
);
const mappedColor = colorValue
? resolveNearestSupportedColor(colorValue)
: null;
const deltas = ast.children.flatMap(child =>
context.toDelta(child, baseOptions).map(delta => {
if (mappedColor) {
delta.attributes = { ...delta.attributes, color: mappedColor };
}
return delta;
})
);
return deltas;
},
});
export const htmlTextLikeElementToDeltaMatcher = HtmlASTToDeltaExtension({
name: 'text-like-element',
match: ast => isTextLikeElement(ast),
@@ -300,6 +339,7 @@ export const htmlBrElementToDeltaMatcher = HtmlASTToDeltaExtension({
export const HtmlInlineToDeltaAdapterExtensions = [
htmlTextToDeltaMatcher,
htmlColorStyleElementToDeltaMatcher,
htmlTextLikeElementToDeltaMatcher,
htmlStrongElementToDeltaMatcher,
htmlItalicElementToDeltaMatcher,

4
blocksuite/affine/inlines/preset/src/adapters/markdown/markdown-inline.ts
View File

@@ -79,11 +79,11 @@ export const markdownListToDeltaMatcher = MarkdownASTToDeltaExtension({
export const markdownHtmlToDeltaMatcher = MarkdownASTToDeltaExtension({
name: 'html',
match: ast => ast.type === 'html',
toDelta: ast => {
toDelta: (ast, context) => {
if (!('value' in ast)) {
return [];
}
return [{ insert: ast.value }];
return context?.htmlToDelta?.(ast.value) ?? [{ insert: ast.value }];
},
});

212
blocksuite/affine/shared/src/adapters/markdown/delta-converter.ts
View File

@@ -3,9 +3,17 @@ import {
type ServiceIdentifier,
} from '@blocksuite/global/di';
import type { DeltaInsert, ExtensionType } from '@blocksuite/store';
import type { Root } from 'hast';
import type { PhrasingContent } from 'mdast';
import rehypeParse from 'rehype-parse';
import { unified } from 'unified';
import type { AffineTextAttributes } from '../../types/index.js';
import { HtmlDeltaConverter } from '../html/delta-converter.js';
import {
rehypeInlineToBlock,
rehypeWrapInlineElements,
} from '../html/rehype-plugins/index.js';
import {
type ASTToDeltaMatcher,
DeltaASTConverter,
@@ -13,6 +21,88 @@ import {
} from '../types/delta-converter.js';
import type { MarkdownAST } from './type.js';
const INLINE_HTML_TAGS = new Set([
'span',
'strong',
'b',
'em',
'i',
'del',
'u',
'mark',
'code',
'ins',
'bdi',
'bdo',
]);
const VOID_HTML_TAGS = new Set([
'area',
'base',
'br',
'col',
'embed',
'hr',
'img',
'input',
'link',
'meta',
'param',
'source',
'track',
'wbr',
]);
const ALLOWED_INLINE_HTML_TAGS = new Set([
...INLINE_HTML_TAGS,
...VOID_HTML_TAGS,
]);
const isHtmlNode = (
node: MarkdownAST
): node is MarkdownAST & { type: 'html'; value: string } =>
node.type === 'html' && 'value' in node && typeof node.value === 'string';
const isTextNode = (
node: MarkdownAST
): node is MarkdownAST & { type: 'text'; value: string } =>
node.type === 'text' && 'value' in node && typeof node.value === 'string';
type HtmlTagInfo =
| { name: string; kind: 'open' | 'self' }
| { name: string; kind: 'close' };
const getHtmlTagInfo = (value: string): HtmlTagInfo | null => {
const closingMatch = value.match(/^<\/([A-Za-z][A-Za-z0-9-]*)\s*>$/);
if (closingMatch) {
return {
name: closingMatch[1].toLowerCase(),
kind: 'close',
};
}
const selfClosingMatch = value.match(
/^<([A-Za-z][A-Za-z0-9-]*)(\s[^>]*)?\/>$/i
);
if (selfClosingMatch) {
return {
name: selfClosingMatch[1].toLowerCase(),
kind: 'self',
};
}
const openingMatch = value.match(/^<([A-Za-z][A-Za-z0-9-]*)(\s[^>]*)?>$/);
if (openingMatch) {
const name = openingMatch[1].toLowerCase();
return {
name,
kind: VOID_HTML_TAGS.has(name) ? 'self' : 'open',
};
}
return null;
};
export type InlineDeltaToMarkdownAdapterMatcher =
InlineDeltaMatcher<PhrasingContent>;
@@ -63,11 +153,30 @@ export class MarkdownDeltaConverter extends DeltaASTConverter<
constructor(
readonly configs: Map<string, string>,
readonly inlineDeltaMatchers: InlineDeltaToMarkdownAdapterMatcher[],
readonly markdownASTToDeltaMatchers: MarkdownASTToDeltaMatcher[]
readonly markdownASTToDeltaMatchers: MarkdownASTToDeltaMatcher[],
readonly htmlDeltaConverter?: HtmlDeltaConverter
) {
super();
}
private _convertHtmlToDelta(
html: string
): DeltaInsert<AffineTextAttributes>[] {
if (!this.htmlDeltaConverter) {
return [{ insert: html }];
}
try {
const processor = unified()
.use(rehypeParse, { fragment: true })
.use(rehypeInlineToBlock)
.use(rehypeWrapInlineElements);
const ast = processor.runSync(processor.parse(html)) as Root;
return this.htmlDeltaConverter.astToDelta(ast, { trim: false });
} catch {
return [{ insert: html }];
}
}
applyTextFormatting(
delta: DeltaInsert<AffineTextAttributes>
): PhrasingContent {
@@ -95,11 +204,110 @@ export class MarkdownDeltaConverter extends DeltaASTConverter<
return mdast;
}
private _mergeInlineHtml(
children: MarkdownAST[],
startIndex: number
): {
endIndex: number;
deltas: DeltaInsert<AffineTextAttributes>[];
} | null {
const startNode = children[startIndex];
if (!isHtmlNode(startNode)) {
return null;
}
const startTag = getHtmlTagInfo(startNode.value);
if (
!startTag ||
startTag.kind !== 'open' ||
!INLINE_HTML_TAGS.has(startTag.name)
) {
return null;
}
const stack = [startTag.name];
let html = startNode.value;
let endIndex = startIndex;
for (let i = startIndex + 1; i < children.length; i++) {
const node = children[i];
if (isHtmlNode(node)) {
const info = getHtmlTagInfo(node.value);
if (!info) {
html += node.value;
continue;
}
if (info.kind === 'open') {
if (!ALLOWED_INLINE_HTML_TAGS.has(info.name)) {
return null;
}
stack.push(info.name);
html += node.value;
continue;
}
if (info.kind === 'self') {
if (!ALLOWED_INLINE_HTML_TAGS.has(info.name)) {
return null;
}
html += node.value;
continue;
}
if (!ALLOWED_INLINE_HTML_TAGS.has(info.name)) {
return null;
}
const last = stack[stack.length - 1];
if (last !== info.name) {
return null;
}
stack.pop();
html += node.value;
endIndex = i;
if (stack.length === 0) {
return {
endIndex,
deltas: this._convertHtmlToDelta(html),
};
}
continue;
}
if (isTextNode(node)) {
html += node.value;
continue;
}
return null;
}
return null;
}
private _astChildrenToDelta(
children: MarkdownAST[]
): DeltaInsert<AffineTextAttributes>[] {
const deltas: DeltaInsert<AffineTextAttributes>[] = [];
for (let i = 0; i < children.length; i++) {
const merged = this._mergeInlineHtml(children, i);
if (merged) {
deltas.push(...merged.deltas);
i = merged.endIndex;
continue;
}
deltas.push(...this.astToDelta(children[i]));
}
return deltas;
}
astToDelta(ast: MarkdownAST): DeltaInsert<AffineTextAttributes>[] {
const context = {
configs: this.configs,
options: Object.create(null),
toDelta: (ast: MarkdownAST) => this.astToDelta(ast),
htmlToDelta: (html: string) => this._convertHtmlToDelta(html),
};
for (const matcher of this.markdownASTToDeltaMatchers) {
if (matcher.match(ast)) {
@@ -107,7 +315,7 @@ export class MarkdownDeltaConverter extends DeltaASTConverter<
}
}
return 'children' in ast
? ast.children.flatMap(child => this.astToDelta(child))
? this._astChildrenToDelta(ast.children as MarkdownAST[])
: [];
}

20
blocksuite/affine/shared/src/adapters/markdown/markdown.ts
View File

@@ -26,6 +26,11 @@ import remarkParse from 'remark-parse';
import remarkStringify from 'remark-stringify';
import { unified } from 'unified';
import {
HtmlASTToDeltaMatcherIdentifier,
HtmlDeltaConverter,
InlineDeltaToHtmlAdapterMatcherIdentifier,
} from '../html/delta-converter.js';
import { type AdapterContext, AdapterFactoryIdentifier } from '../types';
import {
type BlockMarkdownAdapterMatcher,
@@ -184,11 +189,24 @@ export class MarkdownAdapter extends BaseAdapter<Markdown> {
const markdownInlineToDeltaMatchers = Array.from(
provider.getAll(MarkdownASTToDeltaMatcherIdentifier).values()
);
const inlineDeltaToHtmlAdapterMatchers = Array.from(
provider.getAll(InlineDeltaToHtmlAdapterMatcherIdentifier).values()
);
const htmlInlineToDeltaMatchers = Array.from(
provider.getAll(HtmlASTToDeltaMatcherIdentifier).values()
);
const htmlDeltaConverter = new HtmlDeltaConverter(
job.adapterConfigs,
inlineDeltaToHtmlAdapterMatchers,
htmlInlineToDeltaMatchers,
provider
);
this.blockMatchers = blockMatchers;
this.deltaConverter = new MarkdownDeltaConverter(
job.adapterConfigs,
inlineDeltaToMarkdownAdapterMatchers,
markdownInlineToDeltaMatchers
markdownInlineToDeltaMatchers,
htmlDeltaConverter
);
this.preprocessorManager = new MarkdownPreprocessorManager(provider);
}

1
blocksuite/affine/shared/src/adapters/types/delta-converter.ts
View File

@@ -56,6 +56,7 @@ export type ASTToDeltaMatcher<AST> = {
ast: AST,
options?: DeltaASTConverterOptions
) => DeltaInsert<AffineTextAttributes>[];
htmlToDelta?: (html: string) => DeltaInsert<AffineTextAttributes>[];
}
) => DeltaInsert<AffineTextAttributes>[];
};

1
blocksuite/affine/widgets/linked-doc/package.json
View File

@@ -26,6 +26,7 @@
"@toeverything/theme": "^1.1.16",
"@types/lodash-es": "^4.17.12",
"fflate": "^0.8.2",
"js-yaml": "^4.1.1",
"lit": "^3.2.0",
"lodash-es": "^4.17.21",
"mammoth": "^1.11.0",

191
blocksuite/affine/widgets/linked-doc/src/transformers/markdown.ts
View File

@@ -15,10 +15,183 @@ import type {
Store,
Workspace,
} from '@blocksuite/store';
import type { DocMeta } from '@blocksuite/store';
import { extMimeMap, Transformer } from '@blocksuite/store';
import type { AssetMap, ImportedFileEntry, PathBlobIdMap } from './type.js';
import { createAssetsArchive, download, Unzip } from './utils.js';
import { createAssetsArchive, download, parseMatter, Unzip } from './utils.js';
type ParsedFrontmatterMeta = Partial<
Pick<DocMeta, 'title' | 'createDate' | 'updatedDate' | 'tags' | 'favorite'>
>;
const FRONTMATTER_KEYS = {
title: ['title', 'name'],
created: [
'created',
'createdat',
'created_at',
'createddate',
'created_date',
'creationdate',
'date',
'time',
],
updated: [
'updated',
'updatedat',
'updated_at',
'updateddate',
'updated_date',
'modified',
'modifiedat',
'modified_at',
'lastmodified',
'last_modified',
'lastedited',
'last_edited',
'lasteditedtime',
'last_edited_time',
],
tags: ['tags', 'tag', 'categories', 'category', 'labels', 'keywords'],
favorite: ['favorite', 'favourite', 'star', 'starred', 'pinned'],
trash: ['trash', 'trashed', 'deleted', 'archived'],
};
const truthyStrings = new Set(['true', 'yes', 'y', '1', 'on']);
const falsyStrings = new Set(['false', 'no', 'n', '0', 'off']);
function parseBoolean(value: unknown): boolean | undefined {
if (typeof value === 'boolean') return value;
if (typeof value === 'number') {
if (value === 1) return true;
if (value === 0) return false;
}
if (typeof value === 'string') {
const normalized = value.trim().toLowerCase();
if (truthyStrings.has(normalized)) return true;
if (falsyStrings.has(normalized)) return false;
}
return undefined;
}
function parseTimestamp(value: unknown): number | undefined {
if (value && value instanceof Date) {
return value.getTime();
}
if (typeof value === 'number' && Number.isFinite(value)) {
return value > 1e10 ? value : Math.round(value * 1000);
}
if (typeof value === 'string') {
const num = Number(value);
if (!Number.isNaN(num)) {
return num > 1e10 ? num : Math.round(num * 1000);
}
const parsed = Date.parse(value);
if (!Number.isNaN(parsed)) {
return parsed;
}
}
return undefined;
}
function parseTags(value: unknown): string[] | undefined {
if (Array.isArray(value)) {
const tags = value
.map(v => (typeof v === 'string' ? v : String(v)))
.map(v => v.trim())
.filter(Boolean);
return tags.length ? [...new Set(tags)] : undefined;
}
if (typeof value === 'string') {
const tags = value
.split(/[,;]+/)
.map(v => v.trim())
.filter(Boolean);
return tags.length ? [...new Set(tags)] : undefined;
}
return undefined;
}
function buildMetaFromFrontmatter(
data: Record<string, unknown>
): ParsedFrontmatterMeta {
const meta: ParsedFrontmatterMeta = {};
for (const [rawKey, value] of Object.entries(data)) {
const key = rawKey.trim().toLowerCase();
if (FRONTMATTER_KEYS.title.includes(key) && typeof value === 'string') {
const title = value.trim();
if (title) meta.title = title;
continue;
}
if (FRONTMATTER_KEYS.created.includes(key)) {
const timestamp = parseTimestamp(value);
if (timestamp !== undefined) {
meta.createDate = timestamp;
}
continue;
}
if (FRONTMATTER_KEYS.updated.includes(key)) {
const timestamp = parseTimestamp(value);
if (timestamp !== undefined) {
meta.updatedDate = timestamp;
}
continue;
}
if (FRONTMATTER_KEYS.tags.includes(key)) {
const tags = parseTags(value);
if (tags) meta.tags = tags;
continue;
}
if (FRONTMATTER_KEYS.favorite.includes(key)) {
const favorite = parseBoolean(value);
if (favorite !== undefined) {
meta.favorite = favorite;
}
continue;
}
}
return meta;
}
function parseFrontmatter(markdown: string): {
content: string;
meta: ParsedFrontmatterMeta;
} {
try {
const parsed = parseMatter(markdown);
if (!parsed) {
return { content: markdown, meta: {} };
}
const content = parsed.body ?? markdown;
if (Array.isArray(parsed.metadata)) {
return { content: String(content), meta: {} };
}
const meta = buildMetaFromFrontmatter({ ...parsed.metadata });
return { content: String(content), meta };
} catch {
return { content: markdown, meta: {} };
}
}
function applyMetaPatch(
collection: Workspace,
docId: string,
meta: ParsedFrontmatterMeta
) {
const metaPatch: Partial<DocMeta> = {};
if (meta.title) metaPatch.title = meta.title;
if (meta.createDate !== undefined) metaPatch.createDate = meta.createDate;
if (meta.updatedDate !== undefined) metaPatch.updatedDate = meta.updatedDate;
if (meta.tags) metaPatch.tags = meta.tags;
if (meta.favorite !== undefined) metaPatch.favorite = meta.favorite;
if (Object.keys(metaPatch).length) {
collection.meta.setDocMeta(docId, metaPatch);
}
}
function getProvider(extensions: ExtensionType[]) {
const container = new Container();
@@ -153,6 +326,8 @@ async function importMarkdownToDoc({
fileName,
extensions,
}: ImportMarkdownToDocOptions) {
const { content, meta } = parseFrontmatter(markdown);
const preferredTitle = meta.title ?? fileName;
const provider = getProvider(extensions);
const job = new Transformer({
schema,
@@ -164,18 +339,19 @@ async function importMarkdownToDoc({
},
middlewares: [
defaultImageProxyMiddleware,
fileNameMiddleware(fileName),
fileNameMiddleware(preferredTitle),
docLinkBaseURLMiddleware(collection.id),
],
});
const mdAdapter = new MarkdownAdapter(job, provider);
const page = await mdAdapter.toDoc({
file: markdown,
file: content,
assets: job.assetsManager,
});
if (!page) {
return;
}
applyMetaPatch(collection, page.id, meta);
return page.id;
}
@@ -232,6 +408,9 @@ async function importMarkdownZip({
markdownBlobs.map(async markdownFile => {
const { filename, contentBlob, fullPath } = markdownFile;
const fileNameWithoutExt = filename.replace(/\.[^/.]+$/, '');
const markdown = await contentBlob.text();
const { content, meta } = parseFrontmatter(markdown);
const preferredTitle = meta.title ?? fileNameWithoutExt;
const job = new Transformer({
schema,
blobCRUD: collection.blobSync,
@@ -242,7 +421,7 @@ async function importMarkdownZip({
},
middlewares: [
defaultImageProxyMiddleware,
fileNameMiddleware(fileNameWithoutExt),
fileNameMiddleware(preferredTitle),
docLinkBaseURLMiddleware(collection.id),
filePathMiddleware(fullPath),
],
@@ -262,12 +441,12 @@ async function importMarkdownZip({
}
const mdAdapter = new MarkdownAdapter(job, provider);
const markdown = await contentBlob.text();
const doc = await mdAdapter.toDoc({
file: markdown,
file: content,
assets: job.assetsManager,
});
if (doc) {
applyMetaPatch(collection, doc.id, meta);
docIds.push(doc.id);
}
})

12
blocksuite/affine/widgets/linked-doc/src/transformers/utils.ts
View File

@@ -1,5 +1,6 @@
import { extMimeMap, getAssetName } from '@blocksuite/store';
import * as fflate from 'fflate';
import { FAILSAFE_SCHEMA, load as loadYaml } from 'js-yaml';
export class Zip {
private compressed = new Uint8Array();
@@ -208,3 +209,14 @@ export function download(blob: Blob, name: string) {
element.remove();
URL.revokeObjectURL(fileURL);
}
const metaMatcher = /(?<=---)(.*?)(?=---)/ms;
const bodyMatcher = /---.*?---/s;
export const parseMatter = (contents: string) => {
const matterMatch = contents.match(metaMatcher);
if (!matterMatch || !matterMatch[0]) return null;
const metadata = loadYaml(matterMatch[0], { schema: FAILSAFE_SCHEMA });
if (!metadata || typeof metadata !== 'object') return null;
const body = contents.replace(bodyMatcher, '');
return { matter: matterMatch[0], body, metadata };
};

2
package.json
View File

@@ -92,7 +92,7 @@
"vite": "^7.2.7",
"vitest": "^3.2.4"
},
"packageManager": "yarn@4.9.1",
"packageManager": "yarn@4.12.0",
"resolutions": {
"array-buffer-byte-length": "npm:@nolyfill/array-buffer-byte-length@^1",
"array-includes": "npm:@nolyfill/array-includes@^1",

1
packages/common/y-octo/yjs

Submodule packages/common/y-octo/yjs deleted from 7126035d1b

15
yarn.lock
View File

@@ -4118,6 +4118,7 @@ __metadata:
"@toeverything/theme": "npm:^1.1.16"
"@types/lodash-es": "npm:^4.17.12"
fflate: "npm:^0.8.2"
js-yaml: "npm:^4.1.1"
lit: "npm:^3.2.0"
lodash-es: "npm:^4.17.21"
mammoth: "npm:^1.11.0"
@@ -28727,7 +28728,7 @@ __metadata:
languageName: node
linkType: hard
"js-yaml@npm:^4.0.0, js-yaml@npm:^4.1.0":
"js-yaml@npm:^4.0.0, js-yaml@npm:^4.1.0, js-yaml@npm:^4.1.1":
version: 4.1.1
resolution: "js-yaml@npm:4.1.1"
dependencies:
@@ -33851,11 +33852,11 @@ __metadata:
linkType: hard
"qs@npm:^6.11.0, qs@npm:^6.11.2, qs@npm:^6.14.0, qs@npm:^6.7.0":
version: 6.14.0
resolution: "qs@npm:6.14.0"
version: 6.14.1
resolution: "qs@npm:6.14.1"
dependencies:
side-channel: "npm:^1.1.0"
checksum: 10/a60e49bbd51c935a8a4759e7505677b122e23bf392d6535b8fc31c1e447acba2c901235ecb192764013cd2781723dc1f61978b5fdd93cc31d7043d31cdc01974
checksum: 10/34b5ab00a910df432d55180ef39c1d1375e550f098b5ec153b41787f1a6a6d7e5f9495593c3b112b77dbc6709d0ae18e55b82847a4c2bbbb0de1e8ccbb1794c5
languageName: node
linkType: hard
@@ -37442,9 +37443,9 @@ __metadata:
linkType: hard
"tmp@npm:^0.2.0":
version: 0.2.3
resolution: "tmp@npm:0.2.3"
checksum: 10/7b13696787f159c9754793a83aa79a24f1522d47b87462ddb57c18ee93ff26c74cbb2b8d9138f571d2e0e765c728fb2739863a672b280528512c6d83d511c6fa
version: 0.2.5
resolution: "tmp@npm:0.2.5"
checksum: 10/dd4b78b32385eab4899d3ae296007b34482b035b6d73e1201c4a9aede40860e90997a1452c65a2d21aee73d53e93cd167d741c3db4015d90e63b6d568a93d7ec
languageName: node
linkType: hard