Jayden/AFFiNE-Mirror
0
0
Fork 0
mirror of https://github.com/toeverything/AFFiNE.git synced 2026-07-25 22:38:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(core): support ai network search (#9357)

Browse Source 
### What Changed?
- Add `PerplexityProvider` in backend.
- Update session prompt name if user toggle network search mode in chat panel.
- Add experimental flag for AI network search feature.
- Add unit tests and e2e tests.

Search results are streamed and appear word for word:

<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/56f6ec7b-4b21-405f-9612-43e083f6fb84.mov">录屏2024-12-27 18.58.40.mov</video>

Click the little globe icon to manually turn on/off Internet search:
<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/778f1406-bf29-498e-a90d-7dad813392d1.mov">录屏2024-12-27 19.01.16.mov</video>

When there is an image, it will automatically switch to the openai model:

<div class='graphite__hidden'>
          <div>🎥 Video uploaded on Graphite:</div>
            <a href="https://app.graphite.dev/media/video/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov">
              <img src="https://app.graphite.dev/api/v1/graphite/video/thumbnail/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov">
            </a>
          </div>
<video src="https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/sJGviKxfE3Ap685cl5bj/56431d8e-75e1-4d84-ab4a-b6636042cc6a.mov">录屏2024-12-27 19.02.13.mov</video>
This commit is contained in:
akumatus
2025-01-09 04:00:58 +00:00
parent 4f10457815
commit 58ce86533e
49 changed files with 1274 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/server/tests/copilot-provider.spec.ts
+11 -2
View File
@@ -13,6 +13,7 @@ import {
CopilotProviderService,
FalProvider,
OpenAIProvider,
PerplexityProvider,
registerCopilotProvider,
unregisterCopilotProvider,
} from '../src/plugins/copilot/providers';
@@ -47,8 +48,10 @@ const test = ava as TestFn<Tester>;
const isCopilotConfigured =
!!process.env.COPILOT_OPENAI_API_KEY &&
!!process.env.COPILOT_FAL_API_KEY &&
!!process.env.COPILOT_PERPLEXITY_API_KEY &&
process.env.COPILOT_OPENAI_API_KEY !== '1' &&
process.env.COPILOT_FAL_API_KEY !== '1';
process.env.COPILOT_FAL_API_KEY !== '1' &&
process.env.COPILOT_PERPLEXITY_API_KEY !== '1';
const runIfCopilotConfigured = test.macro(
async (
t,
@@ -75,6 +78,9 @@ test.serial.before(async t => {
fal: {
apiKey: process.env.COPILOT_FAL_API_KEY,
},
perplexity: {
apiKey: process.env.COPILOT_PERPLEXITY_API_KEY,
},
},
},
}),
@@ -111,6 +117,7 @@ test.serial.before(async t => {
registerCopilotProvider(OpenAIProvider);
registerCopilotProvider(FalProvider);
registerCopilotProvider(PerplexityProvider);
for (const name of await prompt.listNames()) {
await prompt.delete(name);
@@ -124,6 +131,7 @@ test.serial.before(async t => {
test.after(async _ => {
unregisterCopilotProvider(OpenAIProvider.type);
unregisterCopilotProvider(FalProvider.type);
unregisterCopilotProvider(PerplexityProvider.type);
});
test.after(async t => {
@@ -152,7 +160,6 @@ const checkMDList = (text: string) => {
return false;
}
// eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
const currentIndent = line.match(/^( *)/)?.[0].length!;
if (Number.isNaN(currentIndent) || currentIndent % 2 !== 0) {
return false;
@@ -282,6 +289,8 @@ const actions = [
'Make it longer',
'Make it shorter',
'Continue writing',
'Chat With AFFiNE AI',
'Search With AFFiNE AI',
],
messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
verifier: (t: ExecutionContext<Tester>, result: string) => {
packages/backend/server/tests/copilot.e2e.ts
+85
View File
@@ -16,6 +16,7 @@ import {
CopilotProviderService,
FalProvider,
OpenAIProvider,
PerplexityProvider,
registerCopilotProvider,
unregisterCopilotProvider,
} from '../src/plugins/copilot/providers';
@@ -41,6 +42,7 @@ import {
sse2array,
textToEventStream,
unsplashSearch,
updateCopilotSession,
} from './utils/copilot';
const test = ava as TestFn<{
@@ -63,6 +65,9 @@ test.beforeEach(async t => {
fal: {
apiKey: '1',
},
perplexity: {
apiKey: '1',
},
unsplashKey: process.env.UNSPLASH_ACCESS_KEY || '1',
},
},
@@ -91,6 +96,7 @@ test.beforeEach(async t => {
unregisterCopilotProvider(OpenAIProvider.type);
unregisterCopilotProvider(FalProvider.type);
unregisterCopilotProvider(PerplexityProvider.type);
registerCopilotProvider(MockCopilotTestProvider);
await prompt.set(promptName, 'test', [
@@ -156,6 +162,85 @@ test('should create session correctly', async t => {
}
});
test('should update session correctly', async t => {
const { app } = t.context;
const assertUpdateSession = async (
sessionId: string,
error: string,
asserter = async (x: any) => {
t.truthy(await x, error);
}
) => {
await asserter(updateCopilotSession(app, token, sessionId, promptName));
};
{
const { id: workspaceId } = await createWorkspace(app, token);
const docId = randomUUID();
const sessionId = await createCopilotSession(
app,
token,
workspaceId,
docId,
promptName
);
await assertUpdateSession(
sessionId,
'should be able to update session with cloud workspace that user can access'
);
}
{
const sessionId = await createCopilotSession(
app,
token,
randomUUID(),
randomUUID(),
promptName
);
await assertUpdateSession(
sessionId,
'should be able to update session with local workspace'
);
}
{
const aToken = (await signUp(app, 'test', 'test@affine.pro', '123456'))
.token.token;
const { id: workspaceId } = await createWorkspace(app, aToken);
const inviteId = await inviteUser(
app,
aToken,
workspaceId,
'darksky@affine.pro'
);
await acceptInviteById(app, workspaceId, inviteId, false);
const sessionId = await createCopilotSession(
app,
token,
workspaceId,
randomUUID(),
promptName
);
await assertUpdateSession(
sessionId,
'should able to update session after user have permission'
);
}
{
const sessionId = '123456';
await assertUpdateSession(sessionId, '', async x => {
await t.throwsAsync(
x,
{ instanceOf: Error },
'should not able to update invalid session id'
);
});
}
});
test('should fork session correctly', async t => {
const { app } = t.context;
packages/backend/server/tests/copilot.spec.ts
+125 -1
View File
@@ -16,6 +16,7 @@ import {
registerCopilotProvider,
unregisterCopilotProvider,
} from '../src/plugins/copilot/providers';
import { CitationParser } from '../src/plugins/copilot/providers/perplexity';
import { ChatSessionService } from '../src/plugins/copilot/session';
import {
CopilotCapability,
@@ -68,7 +69,10 @@ test.beforeEach(async t => {
apiKey: process.env.COPILOT_OPENAI_API_KEY ?? '1',
},
fal: {
apiKey: '1',
apiKey: process.env.COPILOT_FAL_API_KEY ?? '1',
},
perplexity: {
apiKey: process.env.COPILOT_PERPLEXITY_API_KEY ?? '1',
},
},
},
@@ -274,6 +278,41 @@ test('should be able to manage chat session', async t => {
}
});
test('should be able to update chat session prompt', async t => {
const { prompt, session } = t.context;
// Set up a prompt to be used in the session
await prompt.set('prompt', 'model', [
{ role: 'system', content: 'hello {{word}}' },
]);
// Create a session
const sessionId = await session.create({
promptName: 'prompt',
docId: 'test',
workspaceId: 'test',
userId,
});
t.truthy(sessionId, 'should create session');
// Update the session
const updatedSessionId = await session.updateSessionPrompt({
sessionId,
promptName: 'Search With AFFiNE AI',
userId,
});
t.is(updatedSessionId, sessionId, 'should update session with same id');
// Verify the session was updated
const updatedSession = await session.get(sessionId);
t.truthy(updatedSession, 'should retrieve updated session');
t.is(
updatedSession?.config.promptName,
'Search With AFFiNE AI',
'should have updated prompt name'
);
});
test('should be able to fork chat session', async t => {
const { auth, prompt, session } = t.context;
@@ -1050,3 +1089,88 @@ test('should be able to run image executor', async t => {
unregisterCopilotProvider(MockCopilotTestProvider.type);
registerCopilotProvider(OpenAIProvider);
});
test('CitationParser should replace citation placeholders with URLs', t => {
const content =
'This is [a] test sentence with [citations [1]] and [[2]] and [3].';
const citations = ['https://example1.com', 'https://example2.com'];
const parser = new CitationParser();
const result = parser.parse(content, citations);
const expected =
'This is [a] test sentence with [citations [[1](https://example1.com)]] and [[2](https://example2.com)] and [3].';
t.is(result, expected);
});
test('CitationParser should replace chunks of citation placeholders with URLs', t => {
const contents = [
'[[]]',
'This is [',
'a] test sentence ',
'with citations [1',
'] and [',
'[2]] and [[',
'3]] and [[4',
']] and [[5]',
'] and [[6]]',
' and [7',
];
const citations = [
'https://example1.com',
'https://example2.com',
'https://example3.com',
'https://example4.com',
'https://example5.com',
'https://example6.com',
'https://example7.com',
];
const parser = new CitationParser();
let result = contents.reduce((acc, current) => {
return acc + parser.parse(current, citations);
}, '');
result += parser.flush();
const expected =
'[[]]This is [a] test sentence with citations [[1](https://example1.com)] and [[2](https://example2.com)] and [[3](https://example3.com)] and [[4](https://example4.com)] and [[5](https://example5.com)] and [[6](https://example6.com)] and [7';
t.is(result, expected);
});
test('CitationParser should not replace citation already with URLs', t => {
const content =
'This is [a] test sentence with citations [1](https://example1.com) and [[2]](https://example2.com) and [[3](https://example3.com)].';
const citations = [
'https://example4.com',
'https://example5.com',
'https://example6.com',
];
const parser = new CitationParser();
const result = parser.parse(content, citations);
const expected = content;
t.is(result, expected);
});
test('CitationParser should not replace chunks of citation already with URLs', t => {
const contents = [
'This is [a] test sentence with citations [1',
'](https://example1.com) and [[2]',
'](https://example2.com) and [[3](https://example3.com)].',
];
const citations = [
'https://example4.com',
'https://example5.com',
'https://example6.com',
];
const parser = new CitationParser();
let result = contents.reduce((acc, current) => {
return acc + parser.parse(current, citations);
}, '');
result += parser.flush();
const expected = contents.join('');
t.is(result, expected);
});
packages/backend/server/tests/utils/copilot.ts
+25
View File
@@ -184,6 +184,31 @@ export async function createCopilotSession(
return res.body.data.createCopilotSession;
}
export async function updateCopilotSession(
app: INestApplication,
userToken: string,
sessionId: string,
promptName: string
): Promise<string> {
const res = await request(app.getHttpServer())
.post(gql)
.auth(userToken, { type: 'bearer' })
.set({ 'x-request-id': 'test', 'x-operation-name': 'test' })
.send({
query: `
mutation updateCopilotSession($options: UpdateChatSessionInput!) {
updateCopilotSession(options: $options)
}
`,
variables: { options: { sessionId, promptName } },
})
.expect(200);
handleGraphQLError(res);
return res.body.data.updateCopilotSession;
}
export async function forkCopilotSession(
app: INestApplication,
userToken: string,