mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-09-24 04:27:27 +08:00
fix(server): missing embedding search (#13401)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Enhanced search functionality to include results from additional "blob" data sources, providing more comprehensive search results. * **Bug Fixes** * Improved messaging to ensure "No results found" is only shown when no relevant results exist across all data sources. * **Tests** * Updated test cases to reflect new keyword contexts, improving validation accuracy for search-related features. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -534,10 +534,16 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca
|
|||||||
'Chat With AFFiNE AI',
|
'Chat With AFFiNE AI',
|
||||||
'Search With AFFiNE AI',
|
'Search With AFFiNE AI',
|
||||||
],
|
],
|
||||||
messages: [{ role: 'user' as const, content: TestAssets.AFFiNE }],
|
messages: [{ role: 'user' as const, content: TestAssets.SSOT }],
|
||||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||||
assertNotWrappedInCodeBlock(t, result);
|
assertNotWrappedInCodeBlock(t, result);
|
||||||
t.assert(result.includes('AFFiNE'), 'should include original keyword');
|
const cleared = result.toLowerCase();
|
||||||
|
t.assert(
|
||||||
|
cleared.includes('single source of truth') ||
|
||||||
|
/single.*source/.test(cleared) ||
|
||||||
|
cleared.includes('ssot'),
|
||||||
|
'should include original keyword'
|
||||||
|
);
|
||||||
},
|
},
|
||||||
type: 'text' as const,
|
type: 'text' as const,
|
||||||
},
|
},
|
||||||
@@ -595,13 +601,17 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca
|
|||||||
messages: [
|
messages: [
|
||||||
{
|
{
|
||||||
role: 'user' as const,
|
role: 'user' as const,
|
||||||
content: TestAssets.AFFiNE,
|
content: TestAssets.SSOT,
|
||||||
params: { language: 'Simplified Chinese' },
|
params: { language: 'Simplified Chinese' },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
verifier: (t: ExecutionContext<Tester>, result: string) => {
|
||||||
assertNotWrappedInCodeBlock(t, result);
|
assertNotWrappedInCodeBlock(t, result);
|
||||||
t.assert(result.includes('AFFiNE'), 'should include keyword');
|
const cleared = result.toLowerCase();
|
||||||
|
t.assert(
|
||||||
|
cleared.includes('单一') || cleared.includes('SSOT'),
|
||||||
|
'explain code result should include keyword'
|
||||||
|
);
|
||||||
},
|
},
|
||||||
type: 'text' as const,
|
type: 'text' as const,
|
||||||
},
|
},
|
||||||
@@ -623,7 +633,7 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca
|
|||||||
content.includes('classroom') ||
|
content.includes('classroom') ||
|
||||||
content.includes('school') ||
|
content.includes('school') ||
|
||||||
content.includes('sky'),
|
content.includes('sky'),
|
||||||
'should include keyword'
|
'explain code result should include keyword'
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
type: 'text' as const,
|
type: 'text' as const,
|
||||||
|
|||||||
@@ -232,7 +232,7 @@ export class CopilotContextService implements OnApplicationBootstrap {
|
|||||||
const embedding = await this.embeddingClient.getEmbedding(content, signal);
|
const embedding = await this.embeddingClient.getEmbedding(content, signal);
|
||||||
if (!embedding) return [];
|
if (!embedding) return [];
|
||||||
|
|
||||||
const [fileChunks, workspaceChunks, scopedWorkspaceChunks] =
|
const [fileChunks, blobChunks, workspaceChunks, scopedWorkspaceChunks] =
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
this.models.copilotWorkspace.matchFileEmbedding(
|
this.models.copilotWorkspace.matchFileEmbedding(
|
||||||
workspaceId,
|
workspaceId,
|
||||||
@@ -240,6 +240,12 @@ export class CopilotContextService implements OnApplicationBootstrap {
|
|||||||
topK * 2,
|
topK * 2,
|
||||||
threshold
|
threshold
|
||||||
),
|
),
|
||||||
|
this.models.copilotWorkspace.matchBlobEmbedding(
|
||||||
|
workspaceId,
|
||||||
|
embedding,
|
||||||
|
topK * 2,
|
||||||
|
threshold
|
||||||
|
),
|
||||||
this.models.copilotContext.matchWorkspaceEmbedding(
|
this.models.copilotContext.matchWorkspaceEmbedding(
|
||||||
embedding,
|
embedding,
|
||||||
workspaceId,
|
workspaceId,
|
||||||
@@ -259,6 +265,7 @@ export class CopilotContextService implements OnApplicationBootstrap {
|
|||||||
|
|
||||||
if (
|
if (
|
||||||
!fileChunks.length &&
|
!fileChunks.length &&
|
||||||
|
!blobChunks.length &&
|
||||||
!workspaceChunks.length &&
|
!workspaceChunks.length &&
|
||||||
!scopedWorkspaceChunks?.length
|
!scopedWorkspaceChunks?.length
|
||||||
) {
|
) {
|
||||||
@@ -267,7 +274,12 @@ export class CopilotContextService implements OnApplicationBootstrap {
|
|||||||
|
|
||||||
return await this.embeddingClient.reRank(
|
return await this.embeddingClient.reRank(
|
||||||
content,
|
content,
|
||||||
[...fileChunks, ...workspaceChunks, ...(scopedWorkspaceChunks || [])],
|
[
|
||||||
|
...fileChunks,
|
||||||
|
...blobChunks,
|
||||||
|
...workspaceChunks,
|
||||||
|
...(scopedWorkspaceChunks || []),
|
||||||
|
],
|
||||||
topK,
|
topK,
|
||||||
signal
|
signal
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -42,12 +42,14 @@ export const buildDocSearchGetter = (
|
|||||||
chunks.filter(c => 'docId' in c),
|
chunks.filter(c => 'docId' in c),
|
||||||
'Doc.Read'
|
'Doc.Read'
|
||||||
);
|
);
|
||||||
|
const blobChunks = chunks.filter(c => 'blobId' in c);
|
||||||
const fileChunks = chunks.filter(c => 'fileId' in c);
|
const fileChunks = chunks.filter(c => 'fileId' in c);
|
||||||
if (contextChunks.length) {
|
if (contextChunks.length) {
|
||||||
fileChunks.push(...contextChunks);
|
fileChunks.push(...contextChunks);
|
||||||
}
|
}
|
||||||
if (!docChunks.length && !fileChunks.length)
|
if (!blobChunks.length && !docChunks.length && !fileChunks.length) {
|
||||||
return `No results found for "${query}".`;
|
return `No results found for "${query}".`;
|
||||||
|
}
|
||||||
|
|
||||||
const docIds = docChunks.map(c => ({
|
const docIds = docChunks.map(c => ({
|
||||||
// oxlint-disable-next-line no-non-null-assertion
|
// oxlint-disable-next-line no-non-null-assertion
|
||||||
@@ -80,6 +82,7 @@ export const buildDocSearchGetter = (
|
|||||||
|
|
||||||
return [
|
return [
|
||||||
...fileChunks.map(clearEmbeddingChunk),
|
...fileChunks.map(clearEmbeddingChunk),
|
||||||
|
...blobChunks.map(clearEmbeddingChunk),
|
||||||
...docChunks.map(c => ({
|
...docChunks.map(c => ({
|
||||||
...c,
|
...c,
|
||||||
...docMetas.get(c.docId),
|
...docMetas.get(c.docId),
|
||||||
|
|||||||
Reference in New Issue
Block a user