From bbdea716860bcda22df9160ca686b3c3732f7d44 Mon Sep 17 00:00:00 2001 From: donteatfriedrice Date: Tue, 22 Apr 2025 14:19:09 +0000 Subject: [PATCH] fix(editor): add footnote url markdown preprocessor to avoid link node parsing (#11888) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: [BS-3282](https://linear.app/affine-design/issue/BS-3282/预处理-footnote-definition-中的-url-避免-markdown-link-node-parsing) --- .../__tests__/adapters/markdown.unit.spec.ts | 45 +++++++++---- .../all/src/adapters/markdown/preprocessor.ts | 2 + .../affine/blocks/bookmark/package.json | 3 + .../adapters/preprocessor.unit.spec.ts | 53 +++++++++++++++ .../blocks/bookmark/src/adapters/extension.ts | 6 +- .../blocks/bookmark/src/adapters/index.ts | 2 +- .../bookmark/src/adapters/markdown/index.ts | 12 ++++ .../src/adapters/{ => markdown}/markdown.ts | 0 .../src/adapters/markdown/preprocessor.ts | 67 +++++++++++++++++++ .../affine/blocks/bookmark/vitest.config.ts | 25 +++++++ yarn.lock | 1 + 11 files changed, 200 insertions(+), 16 deletions(-) create mode 100644 blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts create mode 100644 blocksuite/affine/blocks/bookmark/src/adapters/markdown/index.ts rename blocksuite/affine/blocks/bookmark/src/adapters/{ => markdown}/markdown.ts (100%) create mode 100644 blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts create mode 100644 blocksuite/affine/blocks/bookmark/vitest.config.ts diff --git a/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts b/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts index 1b4a6b3737..a82dff5526 100644 --- a/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts +++ b/blocksuite/affine/all/src/__tests__/adapters/markdown.unit.spec.ts @@ -4028,11 +4028,8 @@ hhh expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot); }); - test('without footnote middleware', async () => { - const markdown = - 'aaa[^1][^2][^3]\n\n[^1]: {"type":"url","url":"https%3A%2F%2Fwww.example.com"}\n\n[^2]: {"type":"doc","docId":"deadbeef"}\n\n[^3]: {"type":"attachment","blobId":"abcdefg","fileName":"test.txt","fileType":"text/plain"}\n'; - - const blockSnapshot: BlockSnapshot = { + describe('footnote', () => { + const createFootnoteBlockSnapshot = (url: string): BlockSnapshot => ({ type: 'block', id: 'matchesReplaceMap[0]', flavour: 'affine:note', @@ -4063,7 +4060,7 @@ hhh label: '1', reference: { type: 'url', - url: 'https://www.example.com', + url, }, }, }, @@ -4100,13 +4097,37 @@ hhh children: [], }, ], - }; - - const mdAdapter = new MarkdownAdapter(createJob(), provider); - const rawBlockSnapshot = await mdAdapter.toBlockSnapshot({ - file: markdown, }); - expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot); + + test('with encoded url', async () => { + const markdown = + 'aaa[^1][^2][^3]\n\n[^1]: {"type":"url","url":"https%3A%2F%2Fwww.example.com"}\n\n[^2]: {"type":"doc","docId":"deadbeef"}\n\n[^3]: {"type":"attachment","blobId":"abcdefg","fileName":"test.txt","fileType":"text/plain"}\n'; + + const blockSnapshot = createFootnoteBlockSnapshot( + 'https://www.example.com' + ); + + const mdAdapter = new MarkdownAdapter(createJob(), provider); + const rawBlockSnapshot = await mdAdapter.toBlockSnapshot({ + file: markdown, + }); + expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot); + }); + + test('with unencoded url', async () => { + const markdown = + 'aaa[^1][^2][^3]\n\n[^1]: {"type":"url","url":"https://www.example.com"}\n\n[^2]: {"type":"doc","docId":"deadbeef"}\n\n[^3]: {"type":"attachment","blobId":"abcdefg","fileName":"test.txt","fileType":"text/plain"}\n'; + + const blockSnapshot = createFootnoteBlockSnapshot( + 'https://www.example.com' + ); + + const mdAdapter = new MarkdownAdapter(createJob(), provider); + const rawBlockSnapshot = await mdAdapter.toBlockSnapshot({ + file: markdown, + }); + expect(nanoidReplacement(rawBlockSnapshot)).toEqual(blockSnapshot); + }); }); test('should not wrap url with angle brackets if it is not a url', async () => { diff --git a/blocksuite/affine/all/src/adapters/markdown/preprocessor.ts b/blocksuite/affine/all/src/adapters/markdown/preprocessor.ts index 48a630f5a6..882277c270 100644 --- a/blocksuite/affine/all/src/adapters/markdown/preprocessor.ts +++ b/blocksuite/affine/all/src/adapters/markdown/preprocessor.ts @@ -1,7 +1,9 @@ +import { BookmarkBlockMarkdownPreprocessorExtension } from '@blocksuite/affine-block-bookmark'; import { CodeMarkdownPreprocessorExtension } from '@blocksuite/affine-block-code'; import { LatexMarkdownPreprocessorExtension } from '@blocksuite/affine-block-latex'; export const defaultMarkdownPreprocessors = [ LatexMarkdownPreprocessorExtension, CodeMarkdownPreprocessorExtension, + BookmarkBlockMarkdownPreprocessorExtension, ]; diff --git a/blocksuite/affine/blocks/bookmark/package.json b/blocksuite/affine/blocks/bookmark/package.json index 565f4f905b..34f01f7780 100644 --- a/blocksuite/affine/blocks/bookmark/package.json +++ b/blocksuite/affine/blocks/bookmark/package.json @@ -30,6 +30,9 @@ "yjs": "^13.6.23", "zod": "^3.23.8" }, + "devDependencies": { + "vitest": "3.1.1" + }, "exports": { ".": "./src/index.ts", "./effects": "./src/effects.ts", diff --git a/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts b/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts new file mode 100644 index 0000000000..8cb3eff1e9 --- /dev/null +++ b/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; + +import { footnoteUrlPreprocessor } from '../../adapters/markdown/preprocessor'; + +describe('footnoteUrlPreprocessor', () => { + it('should encode unencoded URLs in footnote definitions', () => { + const input = + '[^ref]: {"type":"url","url":"https://example.com?param=value"}'; + const expected = + '[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue"}'; + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); + + it('should not encode already encoded URLs', () => { + const input = '[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com"}'; + expect(footnoteUrlPreprocessor(input)).toBe(input); + }); + + it('should handle invalid JSON content', () => { + const input = '[^ref]: {"invalid json"}'; + expect(footnoteUrlPreprocessor(input)).toBe(input); + }); + + it('should handle non-object footnote data', () => { + const input = '[^ref]: "not an object"'; + expect(footnoteUrlPreprocessor(input)).toBe(input); + }); + + it('should handle footnote data without url property', () => { + const input = '[^ref]: {"type":"url"}'; + expect(footnoteUrlPreprocessor(input)).toBe(input); + }); + + it('should handle multiple footnote definitions', () => { + const input = ` +[^ref1]: {"type":"url","url":"https://example1.com"} +[^ref2]: {"type":"url","url":"https://example2.com"} + `.trim(); + const expected = ` +[^ref1]: {"type":"url","url":"https%3A%2F%2Fexample1.com"} +[^ref2]: {"type":"url","url":"https%3A%2F%2Fexample2.com"} + `.trim(); + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); + + it('should handle special characters in URLs', () => { + const input = + '[^ref]: {"type":"url","url":"https://example.com/path with spaces?param=value&another=param"}'; + const expected = + '[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%2Fpath%20with%20spaces%3Fparam%3Dvalue%26another%3Dparam"}'; + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); +}); diff --git a/blocksuite/affine/blocks/bookmark/src/adapters/extension.ts b/blocksuite/affine/blocks/bookmark/src/adapters/extension.ts index 6f784e0284..bee4e44187 100644 --- a/blocksuite/affine/blocks/bookmark/src/adapters/extension.ts +++ b/blocksuite/affine/blocks/bookmark/src/adapters/extension.ts @@ -1,13 +1,13 @@ import type { ExtensionType } from '@blocksuite/store'; import { BookmarkBlockHtmlAdapterExtension } from './html.js'; -import { BookmarkBlockMarkdownAdapterExtension } from './markdown.js'; +import { BookmarkBlockMarkdownAdapterExtensions } from './markdown/index.js'; import { BookmarkBlockNotionHtmlAdapterExtension } from './notion-html.js'; import { BookmarkBlockPlainTextAdapterExtension } from './plain-text.js'; export const BookmarkBlockAdapterExtensions: ExtensionType[] = [ BookmarkBlockHtmlAdapterExtension, - BookmarkBlockMarkdownAdapterExtension, + BookmarkBlockMarkdownAdapterExtensions, BookmarkBlockNotionHtmlAdapterExtension, BookmarkBlockPlainTextAdapterExtension, -]; +].flat(); diff --git a/blocksuite/affine/blocks/bookmark/src/adapters/index.ts b/blocksuite/affine/blocks/bookmark/src/adapters/index.ts index b4dd5a6d2a..d91936594e 100644 --- a/blocksuite/affine/blocks/bookmark/src/adapters/index.ts +++ b/blocksuite/affine/blocks/bookmark/src/adapters/index.ts @@ -1,4 +1,4 @@ export * from './html.js'; -export * from './markdown.js'; +export * from './markdown/index.js'; export * from './notion-html.js'; export * from './plain-text.js'; diff --git a/blocksuite/affine/blocks/bookmark/src/adapters/markdown/index.ts b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/index.ts new file mode 100644 index 0000000000..c125bca650 --- /dev/null +++ b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/index.ts @@ -0,0 +1,12 @@ +import type { ExtensionType } from '@blocksuite/store'; + +import { BookmarkBlockMarkdownAdapterExtension } from './markdown.js'; +import { BookmarkBlockMarkdownPreprocessorExtension } from './preprocessor.js'; + +export * from './markdown.js'; +export * from './preprocessor.js'; + +export const BookmarkBlockMarkdownAdapterExtensions: ExtensionType[] = [ + BookmarkBlockMarkdownPreprocessorExtension, + BookmarkBlockMarkdownAdapterExtension, +]; diff --git a/blocksuite/affine/blocks/bookmark/src/adapters/markdown.ts b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/markdown.ts similarity index 100% rename from blocksuite/affine/blocks/bookmark/src/adapters/markdown.ts rename to blocksuite/affine/blocks/bookmark/src/adapters/markdown/markdown.ts diff --git a/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts new file mode 100644 index 0000000000..2596dbf666 --- /dev/null +++ b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts @@ -0,0 +1,67 @@ +import { + type MarkdownAdapterPreprocessor, + MarkdownPreprocessorExtension, +} from '@blocksuite/affine-shared/adapters'; + +// Check if a URL is already encoded with encodeURIComponent +function isEncoded(uri: string): boolean { + try { + // If decoding produces a different result than the original, + // then the URI contains encoded characters + return uri !== decodeURIComponent(uri); + } catch { + // If decoding fails, the URI contains invalid percent-encoding + return true; + } +} + +// Format footnote definition with consistent spacing +function formatFootnoteDefinition(reference: string, data: object): string { + return `[^${reference}]: ${JSON.stringify(data)}`; +} + +/** + * Preprocessor for footnote url + * We should encode url in footnote definition to avoid markdown link parsing + * + * Example of footnote definition: + * [^ref]: {"type":"url","url":"https://example.com"} + */ +export function footnoteUrlPreprocessor(content: string): string { + // Match footnote definitions with any reference (not just numbers) + // Format: [^reference]: {json_content} + return content.replace( + /\[\^([^\]]+)\]:\s*({[\s\S]*?})/g, + (match, reference, jsonContent) => { + try { + const footnoteData = JSON.parse(jsonContent.trim()); + // If footnoteData is not an object or doesn't have url, return original content + // If the url is already encoded, return original content + if ( + typeof footnoteData !== 'object' || + !footnoteData.url || + isEncoded(footnoteData.url) + ) { + return match; + } + + return formatFootnoteDefinition(reference, { + ...footnoteData, + url: encodeURIComponent(footnoteData.url), + }); + } catch { + // Keep original content if JSON parsing fails + return match; + } + } + ); +} + +const bookmarkBlockPreprocessor: MarkdownAdapterPreprocessor = { + name: 'bookmark-block', + levels: ['block', 'slice', 'doc'], + preprocess: footnoteUrlPreprocessor, +}; + +export const BookmarkBlockMarkdownPreprocessorExtension = + MarkdownPreprocessorExtension(bookmarkBlockPreprocessor); diff --git a/blocksuite/affine/blocks/bookmark/vitest.config.ts b/blocksuite/affine/blocks/bookmark/vitest.config.ts new file mode 100644 index 0000000000..255be530ba --- /dev/null +++ b/blocksuite/affine/blocks/bookmark/vitest.config.ts @@ -0,0 +1,25 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + esbuild: { + target: 'es2018', + }, + test: { + browser: { + enabled: true, + headless: true, + name: 'chromium', + provider: 'playwright', + isolate: false, + providerOptions: {}, + }, + include: ['src/__tests__/**/*.unit.spec.ts'], + testTimeout: 500, + coverage: { + provider: 'istanbul', + reporter: ['lcov'], + reportsDirectory: '../../../.coverage/bookmark', + }, + restoreMocks: true, + }, +}); diff --git a/yarn.lock b/yarn.lock index 2b4d9b1d5b..b60f03594a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2417,6 +2417,7 @@ __metadata: lit: "npm:^3.2.0" minimatch: "npm:^10.0.1" rxjs: "npm:^7.8.1" + vitest: "npm:3.1.1" yjs: "npm:^13.6.23" zod: "npm:^3.23.8" languageName: unknown