feat: use footnote for perplexity search results (#9851)

Support issue [BS-2475](https://linear.app/affine-design/issue/BS-2475).

![截屏2025-01-22 16.49.25.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/49febcdf-e403-4a2e-ba99-da36df34e08c.png)

packages/backend/server/src/__tests__/copilot.spec.ts

@@ -1096,10 +1096,14 @@ test('CitationParser should replace citation placeholders with URLs', t => {
   const citations = ['https://example1.com', 'https://example2.com'];
 
   const parser = new CitationParser();
-  const result = parser.parse(content, citations);
+  const result = parser.parse(content, citations) + parser.end();
+
+  const expected = [
+    'This is [a] test sentence with [citations [^1]] and [^2] and [3].',
+    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
+    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
+  ].join('\n\n');
 
-  const expected =
-    'This is [a] test sentence with [citations [[1](https://example1.com)]] and [[2](https://example2.com)] and [3].';
   t.is(result, expected);
 });
 
@@ -1130,10 +1134,18 @@ test('CitationParser should replace chunks of citation placeholders with URLs',
   let result = contents.reduce((acc, current) => {
     return acc + parser.parse(current, citations);
   }, '');
-  result += parser.flush();
+  result += parser.end();
 
-  const expected =
-    '[[]]This is [a] test sentence with citations [[1](https://example1.com)] and [[2](https://example2.com)] and [[3](https://example3.com)] and [[4](https://example4.com)] and [[5](https://example5.com)] and [[6](https://example6.com)] and [7';
+  const expected = [
+    '[[]]This is [a] test sentence with citations [^1] and [^2] and [^3] and [^4] and [^5] and [^6] and [7',
+    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
+    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
+    `[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
+    `[^4]: {"type":"url","url":"${encodeURIComponent(citations[3])}"}`,
+    `[^5]: {"type":"url","url":"${encodeURIComponent(citations[4])}"}`,
+    `[^6]: {"type":"url","url":"${encodeURIComponent(citations[5])}"}`,
+    `[^7]: {"type":"url","url":"${encodeURIComponent(citations[6])}"}`,
+  ].join('\n\n');
   t.is(result, expected);
 });
 
@@ -1147,9 +1159,14 @@ test('CitationParser should not replace citation already with URLs', t => {
   ];
 
   const parser = new CitationParser();
-  const result = parser.parse(content, citations);
+  const result = parser.parse(content, citations) + parser.end();
 
-  const expected = content;
+  const expected = [
+    content,
+    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
+    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
+    `[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
+  ].join('\n\n');
   t.is(result, expected);
 });
 
@@ -1169,8 +1186,13 @@ test('CitationParser should not replace chunks of citation already with URLs', t
   let result = contents.reduce((acc, current) => {
     return acc + parser.parse(current, citations);
   }, '');
-  result += parser.flush();
+  result += parser.end();
 
-  const expected = contents.join('');
+  const expected = [
+    contents.join(''),
+    `[^1]: {"type":"url","url":"${encodeURIComponent(citations[0])}"}`,
+    `[^2]: {"type":"url","url":"${encodeURIComponent(citations[1])}"}`,
+    `[^3]: {"type":"url","url":"${encodeURIComponent(citations[2])}"}`,
+  ].join('\n\n');
   t.is(result, expected);
 });

packages/backend/server/src/plugins/copilot/providers/perplexity.ts

@@ -63,7 +63,10 @@ export class CitationParser {
 
   private numberToken: string[] = [];
 
+  private citations: string[] = [];
+
   public parse(content: string, citations: string[]) {
+    this.citations = citations;
     let result = '';
     const contentArray = content.split('');
     for (const [index, char] of contentArray.entries()) {
@@ -85,7 +88,7 @@ export class CitationParser {
             cIndex <= citations.length &&
             contentArray[index + 1] !== this.PARENTHESES_OPEN
           ) {
-            const content = `[[${cIndex}](${citations[cIndex - 1]})]`;
+            const content = `[^${cIndex}]`;
             result += content;
             this.resetToken();
           } else {
@@ -116,13 +119,26 @@ export class CitationParser {
     return result;
   }
 
-  public flush() {
-    const content = this.getFullContent();
+  public end() {
+    return this.flush() + this.getFootnotes();
+  }
+
+  private flush() {
+    const content = this.getTokenContent();
     this.resetToken();
     return content;
   }
 
-  private getFullContent() {
+  private getFootnotes() {
+    const footnotes = this.citations.map((citation, index) => {
+      return `[^${index + 1}]: {"type":"url","url":"${encodeURIComponent(
+        citation
+      )}"}`;
+    });
+    return '\n\n' + footnotes.join('\n\n');
+  }
+
+  private getTokenContent() {
     return this.startToken.concat(this.numberToken, this.endToken).join('');
   }
 
@@ -208,7 +224,7 @@ export class PerplexityProvider implements CopilotTextToTextProvider {
         const { content } = data.choices[0].message;
         const { citations } = data;
         let result = parser.parse(content, citations);
-        result += parser.flush();
+        result += parser.end();
         return result;
       }
     } catch (e: any) {
@@ -277,7 +293,7 @@ export class PerplexityProvider implements CopilotTextToTextProvider {
                 }
               },
               flush(controller) {
-                controller.enqueue(parser.flush());
+                controller.enqueue(parser.end());
                 controller.enqueue(null);
               },
             })