feat(server): improve context metadata & matching (#12064)

fix AI-20

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Enhanced file metadata with MIME type, blob ID, and file name across context and workspace, now visible in UI and API.
  - Added workspace-level matching for files and documents with configurable thresholds and workspace scoping in search queries.
  - Introduced a new error type and user-friendly messaging for global workspace context matching failures.

- **Bug Fixes**
  - Improved consistent handling of file MIME types and nullable context IDs for accurate metadata.

- **Documentation**
  - Updated GraphQL schema, queries, and mutations to include new metadata fields, optional parameters, and error types.

- **Style**
  - Added new localization strings for global context matching error messages.

- **Tests**
  - Extended test coverage with new and updated snapshot tests for metadata and matching logic.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
darkskygit
2025-05-14 06:32:29 +00:00
parent 04c5fd6dfc
commit cecf545590
36 changed files with 465 additions and 108 deletions
@@ -1,6 +1,7 @@
import { randomUUID } from 'node:crypto';
import { Injectable } from '@nestjs/common';
import { Transactional } from '@nestjs-cls/transactional';
import { Prisma } from '@prisma/client';
import { CopilotSessionNotFound } from '../base';
@@ -179,9 +180,9 @@ export class CopilotContextModel extends BaseModel {
contextId: string,
topK: number,
threshold: number
): Promise<FileChunkSimilarity[]> {
): Promise<Omit<FileChunkSimilarity, 'blobId' | 'name' | 'mimeType'>[]> {
const similarityChunks = await this.db.$queryRaw<
Array<FileChunkSimilarity>
Array<Omit<FileChunkSimilarity, 'blobId' | 'name' | 'mimeType'>>
>`
SELECT "file_id" as "fileId", "chunk", "content", "embedding" <=> ${embedding}::vector as "distance"
FROM "ai_context_embeddings"
@@ -217,6 +218,7 @@ export class CopilotContextModel extends BaseModel {
});
}
@Transactional()
async matchWorkspaceEmbedding(
embedding: number[],
workspaceId: string,
@@ -232,6 +234,18 @@ export class CopilotContextModel extends BaseModel {
ORDER BY "distance" ASC
LIMIT ${topK};
`;
return similarityChunks.filter(c => Number(c.distance) <= threshold);
const matchedChunks = similarityChunks.filter(
c => Number(c.distance) <= threshold
);
const matchedDocIds = Array.from(new Set(matchedChunks.map(c => c.docId)));
if (!matchDocIds?.length && matchedDocIds.length) {
const ignoredDocs = await this.models.copilotWorkspace.checkIgnoredDocs(
workspaceId,
matchedDocIds
);
return matchedChunks.filter(c => !ignoredDocs.includes(c.docId));
}
return matchedChunks;
}
}