From 4887e1d612616d963457893efae501467d6e5f26 Mon Sep 17 00:00:00 2001 From: donteatfriedrice Date: Fri, 25 Apr 2025 09:17:46 +0000 Subject: [PATCH] refactor(editor): improve footnote URL preprocessor (#11975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: [BS-3302](https://linear.app/affine-design/issue/BS-3302/處理-affine-中的-redos-風險) ## Summary by CodeRabbit - **Bug Fixes** - Improved handling of footnote definitions with varied spacing, special characters, and malformed or incomplete entries, ensuring more reliable processing and display. - **New Features** - Introduced advanced parsing for footnote URLs to improve accuracy and validation of footnote content. - **Tests** - Expanded test coverage for footnote processing to include additional edge cases and mixed content scenarios, enhancing overall robustness. --- .../adapters/preprocessor.unit.spec.ts | 66 +++++ .../src/adapters/markdown/preprocessor.ts | 266 ++++++++++++++---- 2 files changed, 284 insertions(+), 48 deletions(-) diff --git a/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts b/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts index d9e10f3a90..7dcbfb39be 100644 --- a/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts +++ b/blocksuite/affine/blocks/bookmark/src/__tests__/adapters/preprocessor.unit.spec.ts @@ -74,4 +74,70 @@ describe('footnoteUrlPreprocessor', () => { '[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue","favicon":"https%3A%2F%2Fexample.com%2Ficon.png%3Fsize%3Dlarge"}'; expect(footnoteUrlPreprocessor(input)).toBe(expected); }); + + it('should format footnote definition with space', () => { + const input = + '[^ref]:{"type":"url","url":"https://example.com?param=value"}'; + const expected = + '[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue"}'; + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); + + it('should format footnote definition with multiple spaces', () => { + const input = + '[^ref]: {"type":"url","url":"https://example.com?param=value"}'; + const expected = + '[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue"}'; + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); + + it('should handle special characters in reference tags', () => { + const input = + '[^ref-with-special-chars!@#]: {"type":"url","url":"https://example.com"}'; + const expected = + '[^ref-with-special-chars!@#]: {"type":"url","url":"https%3A%2F%2Fexample.com"}'; + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); + + it('should handle incomplete footnote definitions', () => { + const input = '[^ref]:'; + expect(footnoteUrlPreprocessor(input)).toBe(input); + }); + + it('should handle mixed content with non-footnote text', () => { + const input = ` +Some regular text +[^ref]: {"type":"url","url":"https://example.com"} +More regular text + `.trim(); + const expected = ` +Some regular text +[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com"} +More regular text + `.trim(); + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); + + it('should handle empty input', () => { + expect(footnoteUrlPreprocessor('')).toBe(''); + }); + + it('should handle malformed footnote definitions', () => { + const input = '[^ref: {"type":"url","url":"https://example.com"}'; + expect(footnoteUrlPreprocessor(input)).toBe(input); + }); + + it('should handle multiple footnotes with mixed content', () => { + const input = ` +[^ref1]: {"type":"url","url":"https://example1.com"} +Some text in between +[^ref2]: {"type":"url","url":"https://example2.com"} + `.trim(); + const expected = ` +[^ref1]: {"type":"url","url":"https%3A%2F%2Fexample1.com"} +Some text in between +[^ref2]: {"type":"url","url":"https%3A%2F%2Fexample2.com"} + `.trim(); + expect(footnoteUrlPreprocessor(input)).toBe(expected); + }); }); diff --git a/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts index 6520b86c49..13dc5392f3 100644 --- a/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts +++ b/blocksuite/affine/blocks/bookmark/src/adapters/markdown/preprocessor.ts @@ -1,8 +1,28 @@ +import { + type FootNoteReferenceParams, + FootNoteReferenceParamsSchema, +} from '@blocksuite/affine-model'; import { type MarkdownAdapterPreprocessor, MarkdownPreprocessorExtension, } from '@blocksuite/affine-shared/adapters'; +// Types for footnote parser tokens +type Token = { + type: + | 'FOOTNOTE_START' + | 'REFERENCE' + | 'SEPARATOR' + | 'JSON_CONTENT' + | 'NEWLINE'; + value: string; +}; + +type FootnoteDefinition = { + reference: string; + content: FootNoteReferenceParams; +}; + // Check if a URL is already encoded with encodeURIComponent function isEncoded(uri: string): boolean { try { @@ -20,6 +40,202 @@ function formatFootnoteDefinition(reference: string, data: object): string { return `[^${reference}]: ${JSON.stringify(data)}`; } +class FootnoteParser { + private pos: number = 0; + private input: string = ''; + private tokens: Token[] = []; + + // Lexer: Convert input string into tokens + private tokenize(input: string): Token[] { + this.input = input; + this.pos = 0; + this.tokens = []; + + while (this.pos < this.input.length) { + const char = this.input[this.pos]; + + // Handle newlines + if (char === '\n') { + this.tokens.push({ type: 'NEWLINE', value: '\n' }); + this.pos++; + continue; + } + + // Match footnote start [^ + if (char === '[' && this.input[this.pos + 1] === '^') { + this.tokens.push({ type: 'FOOTNOTE_START', value: '[^' }); + this.pos += 2; + continue; + } + + // Match reference content until ] + if ( + this.tokens.length > 0 && + this.tokens[this.tokens.length - 1].type === 'FOOTNOTE_START' + ) { + let reference = ''; + while (this.pos < this.input.length && this.input[this.pos] !== ']') { + reference += this.input[this.pos]; + this.pos++; + } + if (reference) { + this.tokens.push({ type: 'REFERENCE', value: reference }); + } + this.pos++; // Skip the closing ] + continue; + } + + // Match separator ': ' + if (char === ':') { + let nextPos = this.pos + 1; + while (nextPos < this.input.length && this.input[nextPos] === ' ') { + nextPos++; + } + this.tokens.push({ + type: 'SEPARATOR', + value: this.input.slice(this.pos, nextPos), + }); + this.pos = nextPos; + continue; + } + + // Match JSON content + if (char === '{') { + let jsonContent = ''; + let braceCount = 1; + let startPos = this.pos; + + this.pos++; // Move past the opening brace + let inString = false; + let escaped = false; + while (this.pos < this.input.length && braceCount > 0) { + const char = this.input[this.pos]; + if (!inString) { + if (char === '{') braceCount++; + else if (char === '}') braceCount--; + } + if (escaped) { + escaped = false; + } else { + if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = !inString; + } + } + jsonContent += char; + this.pos++; + } + + if (braceCount === 0) { + this.tokens.push({ + type: 'JSON_CONTENT', + value: '{' + jsonContent, + }); + } else { + // Reset position if JSON is malformed + this.pos = startPos; + this.pos++; + } + continue; + } + + // Skip other characters + this.pos++; + } + + return this.tokens; + } + + // Parser: Convert tokens into structured data + private parse(tokens: Token[]): FootnoteDefinition[] { + const footnotes: FootnoteDefinition[] = []; + let currentFootnote: Partial = {}; + let isParsingFootnote = false; + + for (const token of tokens) { + switch (token.type) { + case 'FOOTNOTE_START': + isParsingFootnote = true; + currentFootnote = {}; + break; + + case 'REFERENCE': + if (isParsingFootnote) { + currentFootnote.reference = token.value; + } + break; + + case 'JSON_CONTENT': + if (isParsingFootnote) { + try { + currentFootnote.content = FootNoteReferenceParamsSchema.parse( + JSON.parse(token.value) + ); + footnotes.push(currentFootnote as FootnoteDefinition); + } catch { + // Invalid JSON, ignore this footnote + } + isParsingFootnote = false; + currentFootnote = {}; + } + break; + + case 'NEWLINE': + isParsingFootnote = false; + currentFootnote = {}; + break; + } + } + + return footnotes; + } + + // Process URLs in footnote content + private processUrls(footnote: FootnoteDefinition): FootnoteDefinition { + const content = footnote.content; + if (content.url && !isEncoded(content.url)) { + content.url = encodeURIComponent(content.url); + } + if (content.favicon && !isEncoded(content.favicon)) { + content.favicon = encodeURIComponent(content.favicon); + } + return footnote; + } + + // Main processing function + public process(input: string): string { + // 1. Tokenize the input + const tokens = this.tokenize(input); + + // 2. Parse tokens into footnotes + const footnotes = this.parse(tokens); + + // 3. Process URLs in footnotes + const processedFootnotes = footnotes.map(fn => this.processUrls(fn)); + + // 4. Convert back to original format + const footnoteMap = new Map( + processedFootnotes.map(fn => [ + fn.reference, + formatFootnoteDefinition(fn.reference, fn.content), + ]) + ); + + // 5. Replace in original text + return input + .split('\n') + .map(line => { + const match = line.match(/^\[\^([^\]]+)\]:/); + if (match && footnoteMap.has(match[1])) { + return footnoteMap.get(match[1])!; + } + return line; + }) + .join('\n'); + } +} + /** * Preprocessor for footnote url * We should encode url in footnote definition to avoid markdown link parsing @@ -28,54 +244,8 @@ function formatFootnoteDefinition(reference: string, data: object): string { * [^ref]: {"type":"url","url":"https://example.com"} */ export function footnoteUrlPreprocessor(content: string): string { - // Match footnote definitions with any reference (not just numbers) - // Format: [^reference]: {json_content} - return content.replace( - /\[\^([^\]]+)\]:\s*({[\s\S]*?})/g, - (match, reference, jsonContent) => { - try { - const footnoteData = JSON.parse(jsonContent.trim()); - // Basic validation checks - if (typeof footnoteData !== 'object') { - return match; - } - - if (!footnoteData.url) { - return match; - } - - // Check if URLs are already encoded - const isUrlEncoded = isEncoded(footnoteData.url); - const hasIcon = !!footnoteData.favicon; - const isIconEncoded = hasIcon && isEncoded(footnoteData.favicon); - - // If both URL and icon (if present) are already encoded, return original - if (isUrlEncoded && (!hasIcon || isIconEncoded)) { - return match; - } - - // Create processed data with encoded URLs - const processedData = { - ...footnoteData, - url: isUrlEncoded - ? footnoteData.url - : encodeURIComponent(footnoteData.url), - }; - - // Add encoded favicon if present - if (hasIcon) { - processedData.favicon = isIconEncoded - ? footnoteData.favicon - : encodeURIComponent(footnoteData.favicon); - } - - return formatFootnoteDefinition(reference, processedData); - } catch { - // Keep original content if JSON parsing fails - return match; - } - } - ); + const parser = new FootnoteParser(); + return parser.process(content); } const bookmarkBlockPreprocessor: MarkdownAdapterPreprocessor = {