mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-24 22:09:08 +08:00
refactor(editor): improve footnote URL preprocessor (#11975)
Closes: [BS-3302](https://linear.app/affine-design/issue/BS-3302/處理-affine-中的-redos-風險) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Bug Fixes** - Improved handling of footnote definitions with varied spacing, special characters, and malformed or incomplete entries, ensuring more reliable processing and display. - **New Features** - Introduced advanced parsing for footnote URLs to improve accuracy and validation of footnote content. - **Tests** - Expanded test coverage for footnote processing to include additional edge cases and mixed content scenarios, enhancing overall robustness. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
donteatfriedrice
@@ -74,4 +74,70 @@ describe('footnoteUrlPreprocessor', () => {
|
||||
'[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue","favicon":"https%3A%2F%2Fexample.com%2Ficon.png%3Fsize%3Dlarge"}';
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it('should format footnote definition with space', () => {
|
||||
const input =
|
||||
'[^ref]:{"type":"url","url":"https://example.com?param=value"}';
|
||||
const expected =
|
||||
'[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue"}';
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it('should format footnote definition with multiple spaces', () => {
|
||||
const input =
|
||||
'[^ref]: {"type":"url","url":"https://example.com?param=value"}';
|
||||
const expected =
|
||||
'[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com%3Fparam%3Dvalue"}';
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it('should handle special characters in reference tags', () => {
|
||||
const input =
|
||||
'[^ref-with-special-chars!@#]: {"type":"url","url":"https://example.com"}';
|
||||
const expected =
|
||||
'[^ref-with-special-chars!@#]: {"type":"url","url":"https%3A%2F%2Fexample.com"}';
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it('should handle incomplete footnote definitions', () => {
|
||||
const input = '[^ref]:';
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('should handle mixed content with non-footnote text', () => {
|
||||
const input = `
|
||||
Some regular text
|
||||
[^ref]: {"type":"url","url":"https://example.com"}
|
||||
More regular text
|
||||
`.trim();
|
||||
const expected = `
|
||||
Some regular text
|
||||
[^ref]: {"type":"url","url":"https%3A%2F%2Fexample.com"}
|
||||
More regular text
|
||||
`.trim();
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it('should handle empty input', () => {
|
||||
expect(footnoteUrlPreprocessor('')).toBe('');
|
||||
});
|
||||
|
||||
it('should handle malformed footnote definitions', () => {
|
||||
const input = '[^ref: {"type":"url","url":"https://example.com"}';
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(input);
|
||||
});
|
||||
|
||||
it('should handle multiple footnotes with mixed content', () => {
|
||||
const input = `
|
||||
[^ref1]: {"type":"url","url":"https://example1.com"}
|
||||
Some text in between
|
||||
[^ref2]: {"type":"url","url":"https://example2.com"}
|
||||
`.trim();
|
||||
const expected = `
|
||||
[^ref1]: {"type":"url","url":"https%3A%2F%2Fexample1.com"}
|
||||
Some text in between
|
||||
[^ref2]: {"type":"url","url":"https%3A%2F%2Fexample2.com"}
|
||||
`.trim();
|
||||
expect(footnoteUrlPreprocessor(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,8 +1,28 @@
|
||||
import {
|
||||
type FootNoteReferenceParams,
|
||||
FootNoteReferenceParamsSchema,
|
||||
} from '@blocksuite/affine-model';
|
||||
import {
|
||||
type MarkdownAdapterPreprocessor,
|
||||
MarkdownPreprocessorExtension,
|
||||
} from '@blocksuite/affine-shared/adapters';
|
||||
|
||||
// Types for footnote parser tokens
|
||||
type Token = {
|
||||
type:
|
||||
| 'FOOTNOTE_START'
|
||||
| 'REFERENCE'
|
||||
| 'SEPARATOR'
|
||||
| 'JSON_CONTENT'
|
||||
| 'NEWLINE';
|
||||
value: string;
|
||||
};
|
||||
|
||||
type FootnoteDefinition = {
|
||||
reference: string;
|
||||
content: FootNoteReferenceParams;
|
||||
};
|
||||
|
||||
// Check if a URL is already encoded with encodeURIComponent
|
||||
function isEncoded(uri: string): boolean {
|
||||
try {
|
||||
@@ -20,6 +40,202 @@ function formatFootnoteDefinition(reference: string, data: object): string {
|
||||
return `[^${reference}]: ${JSON.stringify(data)}`;
|
||||
}
|
||||
|
||||
class FootnoteParser {
|
||||
private pos: number = 0;
|
||||
private input: string = '';
|
||||
private tokens: Token[] = [];
|
||||
|
||||
// Lexer: Convert input string into tokens
|
||||
private tokenize(input: string): Token[] {
|
||||
this.input = input;
|
||||
this.pos = 0;
|
||||
this.tokens = [];
|
||||
|
||||
while (this.pos < this.input.length) {
|
||||
const char = this.input[this.pos];
|
||||
|
||||
// Handle newlines
|
||||
if (char === '\n') {
|
||||
this.tokens.push({ type: 'NEWLINE', value: '\n' });
|
||||
this.pos++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Match footnote start [^
|
||||
if (char === '[' && this.input[this.pos + 1] === '^') {
|
||||
this.tokens.push({ type: 'FOOTNOTE_START', value: '[^' });
|
||||
this.pos += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Match reference content until ]
|
||||
if (
|
||||
this.tokens.length > 0 &&
|
||||
this.tokens[this.tokens.length - 1].type === 'FOOTNOTE_START'
|
||||
) {
|
||||
let reference = '';
|
||||
while (this.pos < this.input.length && this.input[this.pos] !== ']') {
|
||||
reference += this.input[this.pos];
|
||||
this.pos++;
|
||||
}
|
||||
if (reference) {
|
||||
this.tokens.push({ type: 'REFERENCE', value: reference });
|
||||
}
|
||||
this.pos++; // Skip the closing ]
|
||||
continue;
|
||||
}
|
||||
|
||||
// Match separator ': '
|
||||
if (char === ':') {
|
||||
let nextPos = this.pos + 1;
|
||||
while (nextPos < this.input.length && this.input[nextPos] === ' ') {
|
||||
nextPos++;
|
||||
}
|
||||
this.tokens.push({
|
||||
type: 'SEPARATOR',
|
||||
value: this.input.slice(this.pos, nextPos),
|
||||
});
|
||||
this.pos = nextPos;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Match JSON content
|
||||
if (char === '{') {
|
||||
let jsonContent = '';
|
||||
let braceCount = 1;
|
||||
let startPos = this.pos;
|
||||
|
||||
this.pos++; // Move past the opening brace
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
while (this.pos < this.input.length && braceCount > 0) {
|
||||
const char = this.input[this.pos];
|
||||
if (!inString) {
|
||||
if (char === '{') braceCount++;
|
||||
else if (char === '}') braceCount--;
|
||||
}
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
} else {
|
||||
if (char === '\\') {
|
||||
escaped = true;
|
||||
} else if (char === '"') {
|
||||
inString = !inString;
|
||||
}
|
||||
}
|
||||
jsonContent += char;
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
if (braceCount === 0) {
|
||||
this.tokens.push({
|
||||
type: 'JSON_CONTENT',
|
||||
value: '{' + jsonContent,
|
||||
});
|
||||
} else {
|
||||
// Reset position if JSON is malformed
|
||||
this.pos = startPos;
|
||||
this.pos++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip other characters
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
return this.tokens;
|
||||
}
|
||||
|
||||
// Parser: Convert tokens into structured data
|
||||
private parse(tokens: Token[]): FootnoteDefinition[] {
|
||||
const footnotes: FootnoteDefinition[] = [];
|
||||
let currentFootnote: Partial<FootnoteDefinition> = {};
|
||||
let isParsingFootnote = false;
|
||||
|
||||
for (const token of tokens) {
|
||||
switch (token.type) {
|
||||
case 'FOOTNOTE_START':
|
||||
isParsingFootnote = true;
|
||||
currentFootnote = {};
|
||||
break;
|
||||
|
||||
case 'REFERENCE':
|
||||
if (isParsingFootnote) {
|
||||
currentFootnote.reference = token.value;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'JSON_CONTENT':
|
||||
if (isParsingFootnote) {
|
||||
try {
|
||||
currentFootnote.content = FootNoteReferenceParamsSchema.parse(
|
||||
JSON.parse(token.value)
|
||||
);
|
||||
footnotes.push(currentFootnote as FootnoteDefinition);
|
||||
} catch {
|
||||
// Invalid JSON, ignore this footnote
|
||||
}
|
||||
isParsingFootnote = false;
|
||||
currentFootnote = {};
|
||||
}
|
||||
break;
|
||||
|
||||
case 'NEWLINE':
|
||||
isParsingFootnote = false;
|
||||
currentFootnote = {};
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return footnotes;
|
||||
}
|
||||
|
||||
// Process URLs in footnote content
|
||||
private processUrls(footnote: FootnoteDefinition): FootnoteDefinition {
|
||||
const content = footnote.content;
|
||||
if (content.url && !isEncoded(content.url)) {
|
||||
content.url = encodeURIComponent(content.url);
|
||||
}
|
||||
if (content.favicon && !isEncoded(content.favicon)) {
|
||||
content.favicon = encodeURIComponent(content.favicon);
|
||||
}
|
||||
return footnote;
|
||||
}
|
||||
|
||||
// Main processing function
|
||||
public process(input: string): string {
|
||||
// 1. Tokenize the input
|
||||
const tokens = this.tokenize(input);
|
||||
|
||||
// 2. Parse tokens into footnotes
|
||||
const footnotes = this.parse(tokens);
|
||||
|
||||
// 3. Process URLs in footnotes
|
||||
const processedFootnotes = footnotes.map(fn => this.processUrls(fn));
|
||||
|
||||
// 4. Convert back to original format
|
||||
const footnoteMap = new Map(
|
||||
processedFootnotes.map(fn => [
|
||||
fn.reference,
|
||||
formatFootnoteDefinition(fn.reference, fn.content),
|
||||
])
|
||||
);
|
||||
|
||||
// 5. Replace in original text
|
||||
return input
|
||||
.split('\n')
|
||||
.map(line => {
|
||||
const match = line.match(/^\[\^([^\]]+)\]:/);
|
||||
if (match && footnoteMap.has(match[1])) {
|
||||
return footnoteMap.get(match[1])!;
|
||||
}
|
||||
return line;
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Preprocessor for footnote url
|
||||
* We should encode url in footnote definition to avoid markdown link parsing
|
||||
@@ -28,54 +244,8 @@ function formatFootnoteDefinition(reference: string, data: object): string {
|
||||
* [^ref]: {"type":"url","url":"https://example.com"}
|
||||
*/
|
||||
export function footnoteUrlPreprocessor(content: string): string {
|
||||
// Match footnote definitions with any reference (not just numbers)
|
||||
// Format: [^reference]: {json_content}
|
||||
return content.replace(
|
||||
/\[\^([^\]]+)\]:\s*({[\s\S]*?})/g,
|
||||
(match, reference, jsonContent) => {
|
||||
try {
|
||||
const footnoteData = JSON.parse(jsonContent.trim());
|
||||
// Basic validation checks
|
||||
if (typeof footnoteData !== 'object') {
|
||||
return match;
|
||||
}
|
||||
|
||||
if (!footnoteData.url) {
|
||||
return match;
|
||||
}
|
||||
|
||||
// Check if URLs are already encoded
|
||||
const isUrlEncoded = isEncoded(footnoteData.url);
|
||||
const hasIcon = !!footnoteData.favicon;
|
||||
const isIconEncoded = hasIcon && isEncoded(footnoteData.favicon);
|
||||
|
||||
// If both URL and icon (if present) are already encoded, return original
|
||||
if (isUrlEncoded && (!hasIcon || isIconEncoded)) {
|
||||
return match;
|
||||
}
|
||||
|
||||
// Create processed data with encoded URLs
|
||||
const processedData = {
|
||||
...footnoteData,
|
||||
url: isUrlEncoded
|
||||
? footnoteData.url
|
||||
: encodeURIComponent(footnoteData.url),
|
||||
};
|
||||
|
||||
// Add encoded favicon if present
|
||||
if (hasIcon) {
|
||||
processedData.favicon = isIconEncoded
|
||||
? footnoteData.favicon
|
||||
: encodeURIComponent(footnoteData.favicon);
|
||||
}
|
||||
|
||||
return formatFootnoteDefinition(reference, processedData);
|
||||
} catch {
|
||||
// Keep original content if JSON parsing fails
|
||||
return match;
|
||||
}
|
||||
}
|
||||
);
|
||||
const parser = new FootnoteParser();
|
||||
return parser.process(content);
|
||||
}
|
||||
|
||||
const bookmarkBlockPreprocessor: MarkdownAdapterPreprocessor = {
|
||||
|
||||
Reference in New Issue
Block a user