feat(server): improve context metadata & matching (#12064)

fix AI-20

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Enhanced file metadata with MIME type, blob ID, and file name across context and workspace, now visible in UI and API.
  - Added workspace-level matching for files and documents with configurable thresholds and workspace scoping in search queries.
  - Introduced a new error type and user-friendly messaging for global workspace context matching failures.

- **Bug Fixes**
  - Improved consistent handling of file MIME types and nullable context IDs for accurate metadata.

- **Documentation**
  - Updated GraphQL schema, queries, and mutations to include new metadata fields, optional parameters, and error types.

- **Style**
  - Added new localization strings for global context matching error messages.

- **Tests**
  - Extended test coverage with new and updated snapshot tests for metadata and matching logic.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/backend/server/src/__tests__/copilot.spec.ts

@@ -1316,7 +1316,11 @@ test('should be able to manage context', async t => {
     // file record
     {
       await storage.put(userId, session.workspaceId, 'blob', buffer);
-      const file = await session.addFile('blob', 'sample.pdf');
+      const file = await session.addFile(
+        'blob',
+        'sample.pdf',
+        'application/pdf'
+      );
 
       const handler = Sinon.spy(event, 'emit');
 
@@ -1345,7 +1349,7 @@ test('should be able to manage context', async t => {
         'should list file id'
       );
 
-      const result = await session.matchFileChunks('test', 1, undefined, 1);
+      const result = await session.matchFiles('test', 1, undefined, 1);
       t.is(result.length, 1, 'should match context');
       t.is(result[0].fileId, file.id, 'should match file id');
     }
@@ -1499,13 +1503,13 @@ test('should be able to manage workspace embedding', async t => {
     });
     const contextSession = await context.create(sessionId);
 
-    const ret = await contextSession.matchFileChunks('test', 1, undefined, 1);
+    const ret = await contextSession.matchFiles('test', 1, undefined, 1);
     t.is(ret.length, 1, 'should match workspace context');
     t.is(ret[0].content, 'content', 'should match content');
 
     await workspace.update(ws.id, { enableDocEmbedding: false });
 
-    const ret2 = await contextSession.matchFileChunks('test', 1, undefined, 1);
+    const ret2 = await contextSession.matchFiles('test', 1, undefined, 1);
     t.is(ret2.length, 0, 'should not match workspace context');
   }
 });

packages/backend/server/src/__tests__/models/__snapshots__/copilot-context.spec.ts.md

@@ -6,6 +6,17 @@ Generated by [AVA](https://avajs.dev).
 
 ## should insert embedding by doc id
 
+> should match file embedding
+
+    [
+      {
+        chunk: 0,
+        content: 'content',
+        distance: 0,
+        fileId: 'file-id',
+      },
+    ]
+
 > should return empty array when embedding deleted
 
     []

packages/backend/server/src/__tests__/models/__snapshots__/copilot-workspace.spec.ts.md

@@ -54,3 +54,18 @@ Generated by [AVA](https://avajs.dev).
         docId: 'new_doc',
       },
     ]
+
+## should insert and search embedding
+
+> should match workspace file embedding
+
+    [
+      {
+        blobId: 'blob1',
+        chunk: 0,
+        content: 'content',
+        distance: 0,
+        mimeType: 'text/plain',
+        name: 'file1',
+      },
+    ]

packages/backend/server/src/__tests__/models/copilot-context.spec.ts

@@ -111,8 +111,7 @@ test('should insert embedding by doc id', async t => {
         1,
         1
       );
-      t.is(ret.length, 1);
-      t.is(ret[0].content, 'content');
+      t.snapshot(ret, 'should match file embedding');
     }
 
     {

packages/backend/server/src/__tests__/models/copilot-workspace.spec.ts

@@ -107,6 +107,7 @@ test('should insert and search embedding', async t => {
   {
     const { fileId } = await t.context.copilotWorkspace.addFile(workspace.id, {
       fileName: 'file1',
+      blobId: 'blob1',
       mimeType: 'text/plain',
       size: 1,
     });
@@ -129,8 +130,10 @@ test('should insert and search embedding', async t => {
         1,
         1
       );
-      t.is(ret.length, 1);
-      t.is(ret[0].content, 'content');
+      t.snapshot(
+        cleanObject(ret, ['fileId']),
+        'should match workspace file embedding'
+      );
     }
   }
 });

packages/backend/server/src/base/error/def.ts

@@ -714,6 +714,12 @@ export const USER_FRIENDLY_ERRORS = {
     message: ({ contextId, content, message }) =>
       `Failed to match context ${contextId} with "${escape(content)}": ${message}`,
   },
+  copilot_failed_to_match_global_context: {
+    type: 'internal_server_error',
+    args: { workspaceId: 'string', content: 'string', message: 'string' },
+    message: ({ workspaceId, content, message }) =>
+      `Failed to match context in workspace ${workspaceId} with "${escape(content)}": ${message}`,
+  },
   copilot_embedding_disabled: {
     type: 'action_forbidden',
     message: `Embedding feature is disabled, please contact the administrator to enable it in the workspace settings.`,

packages/backend/server/src/base/error/errors.gen.ts

@@ -780,6 +780,18 @@ export class CopilotFailedToMatchContext extends UserFriendlyError {
     super('internal_server_error', 'copilot_failed_to_match_context', message, args);
   }
 }
+@ObjectType()
+class CopilotFailedToMatchGlobalContextDataType {
+  @Field() workspaceId!: string
+  @Field() content!: string
+  @Field() message!: string
+}
+
+export class CopilotFailedToMatchGlobalContext extends UserFriendlyError {
+  constructor(args: CopilotFailedToMatchGlobalContextDataType, message?: string | ((args: CopilotFailedToMatchGlobalContextDataType) => string)) {
+    super('internal_server_error', 'copilot_failed_to_match_global_context', message, args);
+  }
+}
 
 export class CopilotEmbeddingDisabled extends UserFriendlyError {
   constructor(message?: string) {
@@ -1078,6 +1090,7 @@ export enum ErrorNames {
   COPILOT_CONTEXT_FILE_NOT_SUPPORTED,
   COPILOT_FAILED_TO_MODIFY_CONTEXT,
   COPILOT_FAILED_TO_MATCH_CONTEXT,
+  COPILOT_FAILED_TO_MATCH_GLOBAL_CONTEXT,
   COPILOT_EMBEDDING_DISABLED,
   COPILOT_EMBEDDING_UNAVAILABLE,
   COPILOT_TRANSCRIPTION_JOB_EXISTS,
@@ -1114,5 +1127,5 @@ registerEnumType(ErrorNames, {
 export const ErrorDataUnionType = createUnionType({
   name: 'ErrorDataUnion',
   types: () =>
-    [GraphqlBadRequestDataType, HttpRequestErrorDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType] as const,
+    [GraphqlBadRequestDataType, HttpRequestErrorDataType, QueryTooLongDataType, ValidationErrorDataType, WrongSignInCredentialsDataType, UnknownOauthProviderDataType, InvalidOauthCallbackCodeDataType, MissingOauthQueryParameterDataType, InvalidEmailDataType, InvalidPasswordLengthDataType, WorkspacePermissionNotFoundDataType, SpaceNotFoundDataType, MemberNotFoundInSpaceDataType, NotInSpaceDataType, AlreadyInSpaceDataType, SpaceAccessDeniedDataType, SpaceOwnerNotFoundDataType, SpaceShouldHaveOnlyOneOwnerDataType, DocNotFoundDataType, DocActionDeniedDataType, DocUpdateBlockedDataType, VersionRejectedDataType, InvalidHistoryTimestampDataType, DocHistoryNotFoundDataType, BlobNotFoundDataType, ExpectToGrantDocUserRolesDataType, ExpectToRevokeDocUserRolesDataType, ExpectToUpdateDocUserRoleDataType, NoMoreSeatDataType, UnsupportedSubscriptionPlanDataType, SubscriptionAlreadyExistsDataType, SubscriptionNotExistsDataType, SameSubscriptionRecurringDataType, SubscriptionPlanNotFoundDataType, CopilotDocNotFoundDataType, CopilotMessageNotFoundDataType, CopilotPromptNotFoundDataType, CopilotProviderSideErrorDataType, CopilotInvalidContextDataType, CopilotContextFileNotSupportedDataType, CopilotFailedToModifyContextDataType, CopilotFailedToMatchContextDataType, CopilotFailedToMatchGlobalContextDataType, CopilotFailedToAddWorkspaceFileEmbeddingDataType, RuntimeConfigNotFoundDataType, InvalidRuntimeConfigTypeDataType, InvalidLicenseToActivateDataType, InvalidLicenseUpdateParamsDataType, UnsupportedClientVersionDataType, MentionUserDocAccessDeniedDataType] as const,
 });

packages/backend/server/src/models/common/copilot.ts

@@ -46,6 +46,7 @@ export const ContextFileSchema = z.object({
   id: z.string(),
   chunkSize: z.number(),
   name: z.string(),
+  mimeType: z.string().optional(),
   status: ContextEmbedStatusSchema,
   error: z.string().nullable(),
   blobId: z.string(),
@@ -100,6 +101,9 @@ export type ChunkSimilarity = {
 
 export type FileChunkSimilarity = ChunkSimilarity & {
   fileId: string;
+  blobId: string;
+  name: string;
+  mimeType: string;
 };
 
 export type DocChunkSimilarity = ChunkSimilarity & {
@@ -108,6 +112,7 @@ export type DocChunkSimilarity = ChunkSimilarity & {
 
 export const CopilotWorkspaceFileSchema = z.object({
   fileName: z.string(),
+  blobId: z.string(),
   mimeType: z.string(),
   size: z.number(),
 });

packages/backend/server/src/models/copilot-context.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from 'node:crypto';
 
 import { Injectable } from '@nestjs/common';
+import { Transactional } from '@nestjs-cls/transactional';
 import { Prisma } from '@prisma/client';
 
 import { CopilotSessionNotFound } from '../base';
@@ -179,9 +180,9 @@ export class CopilotContextModel extends BaseModel {
     contextId: string,
     topK: number,
     threshold: number
-  ): Promise<FileChunkSimilarity[]> {
+  ): Promise<Omit<FileChunkSimilarity, 'blobId' | 'name' | 'mimeType'>[]> {
     const similarityChunks = await this.db.$queryRaw<
-      Array<FileChunkSimilarity>
+      Array<Omit<FileChunkSimilarity, 'blobId' | 'name' | 'mimeType'>>
     >`
       SELECT "file_id" as "fileId", "chunk", "content", "embedding" <=> ${embedding}::vector as "distance" 
       FROM "ai_context_embeddings"
@@ -217,6 +218,7 @@ export class CopilotContextModel extends BaseModel {
     });
   }
 
+  @Transactional()
   async matchWorkspaceEmbedding(
     embedding: number[],
     workspaceId: string,
@@ -232,6 +234,18 @@ export class CopilotContextModel extends BaseModel {
       ORDER BY "distance" ASC
       LIMIT ${topK};
     `;
-    return similarityChunks.filter(c => Number(c.distance) <= threshold);
+
+    const matchedChunks = similarityChunks.filter(
+      c => Number(c.distance) <= threshold
+    );
+    const matchedDocIds = Array.from(new Set(matchedChunks.map(c => c.docId)));
+    if (!matchDocIds?.length && matchedDocIds.length) {
+      const ignoredDocs = await this.models.copilotWorkspace.checkIgnoredDocs(
+        workspaceId,
+        matchedDocIds
+      );
+      return matchedChunks.filter(c => !ignoredDocs.includes(c.docId));
+    }
+    return matchedChunks;
   }
 }

packages/backend/server/src/models/copilot-workspace.ts

@@ -16,6 +16,25 @@ import type {
 
 @Injectable()
 export class CopilotWorkspaceConfigModel extends BaseModel {
+  @Transactional()
+  private async listIgnoredDocIds(
+    workspaceId: string,
+    options?: PaginationInput
+  ) {
+    return await this.db.aiWorkspaceIgnoredDocs.findMany({
+      where: {
+        workspaceId,
+      },
+      select: {
+        docId: true,
+        createdAt: true,
+      },
+      orderBy: { createdAt: 'desc' },
+      skip: options?.offset,
+      take: options?.first,
+    });
+  }
+
   @Transactional()
   async updateIgnoredDocs(
     workspaceId: string,
@@ -23,7 +42,7 @@ export class CopilotWorkspaceConfigModel extends BaseModel {
     remove: string[] = []
   ) {
     const removed = new Set(remove);
-    const ignored = await this.listIgnoredDocs(workspaceId).then(
+    const ignored = await this.listIgnoredDocIds(workspaceId).then(
       r => new Set(r.map(r => r.docId).filter(id => !removed.has(id)))
     );
     const added = add.filter(id => !ignored.has(id));
@@ -51,25 +70,11 @@ export class CopilotWorkspaceConfigModel extends BaseModel {
     return added.length + ignored.size;
   }
 
-  @Transactional()
   async listIgnoredDocs(
     workspaceId: string,
-    options?: {
-      includeRead?: boolean;
-    } & PaginationInput
+    options?: PaginationInput
   ): Promise<IgnoredDoc[]> {
-    const row = await this.db.aiWorkspaceIgnoredDocs.findMany({
-      where: {
-        workspaceId,
-      },
-      select: {
-        docId: true,
-        createdAt: true,
-      },
-      orderBy: { createdAt: 'desc' },
-      skip: options?.offset,
-      take: options?.first,
-    });
+    const row = await this.listIgnoredDocIds(workspaceId, options);
     const ids = row.map(r => ({ workspaceId, docId: r.docId }));
     const docs = await this.models.doc.findMetas(ids);
     const docsMap = new Map(
@@ -106,7 +111,7 @@ export class CopilotWorkspaceConfigModel extends BaseModel {
 
   @Transactional()
   async checkIgnoredDocs(workspaceId: string, docIds: string[]) {
-    const ignored = await this.listIgnoredDocs(workspaceId).then(
+    const ignored = await this.listIgnoredDocIds(workspaceId).then(
       r => new Set(r.map(r => r.docId))
     );
 
@@ -214,9 +219,19 @@ export class CopilotWorkspaceConfigModel extends BaseModel {
     const similarityChunks = await this.db.$queryRaw<
       Array<FileChunkSimilarity>
     >`
-      SELECT "file_id" as "fileId", "chunk", "content", "embedding" <=> ${embedding}::vector as "distance" 
-      FROM "ai_workspace_file_embeddings"
-      WHERE workspace_id = ${workspaceId}
+      SELECT
+        e."file_id" as "fileId",
+        f."file_name" as "name",
+        f."blob_id" as "blobId",
+        f."mime_type" as "mimeType",
+        e."chunk",
+        e."content",
+        e."embedding" <=> ${embedding}::vector as "distance" 
+      FROM "ai_workspace_file_embeddings" e
+      JOIN "ai_workspace_files" f
+        ON e."workspace_id" = f."workspace_id"
+        AND e."file_id" = f."file_id"
+      WHERE e.workspace_id = ${workspaceId}
       ORDER BY "distance" ASC
       LIMIT ${topK};
     `;

packages/backend/server/src/plugins/copilot/context/resolver.ts

@@ -23,6 +23,7 @@ import {
   CallMetric,
   CopilotEmbeddingUnavailable,
   CopilotFailedToMatchContext,
+  CopilotFailedToMatchGlobalContext,
   CopilotFailedToModifyContext,
   CopilotSessionNotFound,
   EventBus,
@@ -117,8 +118,8 @@ class RemoveContextFileInput {
 
 @ObjectType('CopilotContext')
 export class CopilotContextType {
-  @Field(() => ID)
-  id!: string;
+  @Field(() => ID, { nullable: true })
+  id!: string | undefined;
 
   @Field(() => String)
   workspaceId!: string;
@@ -169,6 +170,9 @@ class CopilotContextFile implements ContextFile {
   @Field(() => String)
   name!: string;
 
+  @Field(() => String)
+  mimeType!: string;
+
   @Field(() => SafeIntResolver)
   chunkSize!: number;
 
@@ -190,6 +194,15 @@ class ContextMatchedFileChunk implements FileChunkSimilarity {
   @Field(() => String)
   fileId!: string;
 
+  @Field(() => String)
+  blobId!: string;
+
+  @Field(() => String)
+  name!: string;
+
+  @Field(() => String)
+  mimeType!: string;
+
   @Field(() => SafeIntResolver)
   chunk!: number;
 
@@ -283,6 +296,15 @@ export class CopilotContextRootResolver {
       }
     }
 
+    if (copilot.workspaceId) {
+      return [
+        {
+          id: undefined,
+          workspaceId: copilot.workspaceId,
+        },
+      ];
+    }
+
     return [];
   }
 
@@ -387,6 +409,9 @@ export class CopilotContextResolver {
   async collections(
     @Parent() context: CopilotContextType
   ): Promise<CopilotContextCategory[]> {
+    if (!context.id) {
+      return [];
+    }
     const session = await this.context.get(context.id);
     const collections = session.collections;
     await this.models.copilotContext.mergeDocStatus(
@@ -404,6 +429,9 @@ export class CopilotContextResolver {
   async tags(
     @Parent() context: CopilotContextType
   ): Promise<CopilotContextCategory[]> {
+    if (!context.id) {
+      return [];
+    }
     const session = await this.context.get(context.id);
     const tags = session.tags;
     await this.models.copilotContext.mergeDocStatus(
@@ -419,6 +447,9 @@ export class CopilotContextResolver {
   })
   @CallMetric('ai', 'context_file_list')
   async docs(@Parent() context: CopilotContextType): Promise<CopilotDocType[]> {
+    if (!context.id) {
+      return [];
+    }
     const session = await this.context.get(context.id);
     const docs = session.docs;
     await this.models.copilotContext.mergeDocStatus(session.workspaceId, docs);
@@ -433,6 +464,9 @@ export class CopilotContextResolver {
   async files(
     @Parent() context: CopilotContextType
   ): Promise<CopilotContextFile[]> {
+    if (!context.id) {
+      return [];
+    }
     const session = await this.context.get(context.id);
     return session.files;
   }
@@ -593,7 +627,11 @@ export class CopilotContextResolver {
     const session = await this.context.get(options.contextId);
 
     try {
-      const file = await session.addFile(options.blobId, content.filename);
+      const file = await session.addFile(
+        options.blobId,
+        content.filename,
+        content.mimetype
+      );
 
       const buffer = await readStream(content.createReadStream());
       await this.storage.put(
@@ -664,6 +702,8 @@ export class CopilotContextResolver {
     @Args('content') content: string,
     @Args('limit', { type: () => SafeIntResolver, nullable: true })
     limit?: number,
+    @Args('scopedThreshold', { type: () => Float, nullable: true })
+    scopedThreshold?: number,
     @Args('threshold', { type: () => Float, nullable: true })
     threshold?: number
   ): Promise<ContextMatchedFileChunk[]> {
@@ -671,22 +711,46 @@ export class CopilotContextResolver {
       return [];
     }
 
-    const session = await this.context.get(context.id);
-
     try {
-      return await session.matchFileChunks(
+      if (!context.id) {
+        return await this.context.matchWorkspaceFiles(
+          context.workspaceId,
+          content,
+          limit,
+          this.getSignal(ctx.req),
+          threshold
+        );
+      }
+
+      const session = await this.context.get(context.id);
+      return await session.matchFiles(
         content,
         limit,
         this.getSignal(ctx.req),
+        scopedThreshold,
         threshold
       );
     } catch (e: any) {
-      throw new CopilotFailedToMatchContext({
-        contextId: context.id,
-        // don't record the large content
-        content: content.slice(0, 512),
-        message: e.message,
-      });
+      // passthrough user friendly error
+      if (e instanceof UserFriendlyError) {
+        throw e;
+      }
+
+      if (context.id) {
+        throw new CopilotFailedToMatchContext({
+          contextId: context.id,
+          // don't record the large content
+          content: content.slice(0, 512),
+          message: e.message,
+        });
+      } else {
+        throw new CopilotFailedToMatchGlobalContext({
+          workspaceId: context.workspaceId,
+          // don't record the large content
+          content: content.slice(0, 512),
+          message: e.message,
+        });
+      }
     }
   }
 
@@ -711,20 +775,38 @@ export class CopilotContextResolver {
     }
 
     try {
-      const session = await this.context.get(context.id);
       await this.ac
         .user(user.id)
-        .workspace(session.workspaceId)
+        .workspace(context.workspaceId)
         .allowLocal()
         .assert('Workspace.Copilot');
       const allowEmbedding = await this.models.workspace.allowEmbedding(
-        session.workspaceId
+        context.workspaceId
       );
       if (!allowEmbedding) {
         return [];
       }
 
-      const chunks = await session.matchWorkspaceChunks(
+      if (!context.id) {
+        return await this.context.matchWorkspaceDocs(
+          context.workspaceId,
+          content,
+          limit,
+          this.getSignal(ctx.req),
+          threshold
+        );
+      }
+
+      const session = await this.context.get(context.id);
+      if (session.workspaceId !== context.workspaceId) {
+        throw new CopilotFailedToMatchContext({
+          contextId: context.id,
+          // don't record the large content
+          content: content.slice(0, 512),
+          message: 'context not in the same workspace',
+        });
+      }
+      const chunks = await session.matchWorkspaceDocs(
         content,
         limit,
         this.getSignal(ctx.req),
@@ -748,12 +830,22 @@ export class CopilotContextResolver {
       if (e instanceof UserFriendlyError) {
         throw e;
       }
-      throw new CopilotFailedToMatchContext({
-        contextId: context.id,
-        // don't record the large content
-        content: content.slice(0, 512),
-        message: e.message,
-      });
+
+      if (context.id) {
+        throw new CopilotFailedToMatchContext({
+          contextId: context.id,
+          // don't record the large content
+          content: content.slice(0, 512),
+          message: e.message,
+        });
+      } else {
+        throw new CopilotFailedToMatchGlobalContext({
+          workspaceId: context.workspaceId,
+          // don't record the large content
+          content: content.slice(0, 512),
+          message: e.message,
+        });
+      }
     }
   }
 }

packages/backend/server/src/plugins/copilot/context/service.ts

@@ -148,6 +148,48 @@ export class CopilotContextService implements OnApplicationBootstrap {
     return null;
   }
 
+  async matchWorkspaceFiles(
+    workspaceId: string,
+    content: string,
+    topK: number = 5,
+    signal?: AbortSignal,
+    threshold: number = 0.5
+  ) {
+    if (!this.embeddingClient) return [];
+    const embedding = await this.embeddingClient.getEmbedding(content, signal);
+    if (!embedding) return [];
+
+    const chunks = await this.models.copilotWorkspace.matchFileEmbedding(
+      workspaceId,
+      embedding,
+      topK * 2,
+      threshold
+    );
+
+    return this.embeddingClient.reRank(content, chunks, topK, signal);
+  }
+
+  async matchWorkspaceDocs(
+    workspaceId: string,
+    content: string,
+    topK: number = 5,
+    signal?: AbortSignal,
+    threshold: number = 0.5
+  ) {
+    if (!this.embeddingClient) return [];
+    const embedding = await this.embeddingClient.getEmbedding(content, signal);
+    if (!embedding) return [];
+
+    const workspace = await this.models.copilotContext.matchWorkspaceEmbedding(
+      embedding,
+      workspaceId,
+      topK * 2,
+      threshold
+    );
+
+    return this.embeddingClient.reRank(content, workspace, topK);
+  }
+
   @OnEvent('workspace.doc.embed.failed')
   async onDocEmbedFailed({
     contextId,

packages/backend/server/src/plugins/copilot/context/session.ts

@@ -52,12 +52,16 @@ export class ContextSession implements AsyncDisposable {
   }
 
   get files() {
-    return this.config.files.map(f => ({ ...f }));
+    return this.config.files.map(f => this.fulfillFile(f));
   }
 
   get docIds() {
     return Array.from(
-      new Set([this.config.docs, this.config.categories].flat().map(d => d.id))
+      new Set(
+        [this.config.docs, this.config.categories.flatMap(c => c.docs)]
+          .flat()
+          .map(d => d.id)
+      )
     );
   }
 
@@ -136,14 +140,25 @@ export class ContextSession implements AsyncDisposable {
     return true;
   }
 
-  async addFile(blobId: string, name: string): Promise<ContextFile> {
+  private fulfillFile(file: ContextFile): Required<ContextFile> {
+    return {
+      ...file,
+      mimeType: file.mimeType || 'application/octet-stream',
+    };
+  }
+
+  async addFile(
+    blobId: string,
+    name: string,
+    mimeType: string
+  ): Promise<Required<ContextFile>> {
     let fileId = nanoid();
     const existsBlob = this.config.files.find(f => f.blobId === blobId);
     if (existsBlob) {
       // use exists file id if the blob exists
       // we assume that the file content pointed to by the same blobId is consistent.
       if (existsBlob.status === ContextEmbedStatus.finished) {
-        return existsBlob;
+        return this.fulfillFile(existsBlob);
       }
       fileId = existsBlob.id;
     } else {
@@ -152,11 +167,12 @@ export class ContextSession implements AsyncDisposable {
         blobId,
         chunkSize: 0,
         name,
+        mimeType,
         error: null,
         createdAt: Date.now(),
       }));
     }
-    return this.getFile(fileId) as ContextFile;
+    return this.fulfillFile(this.getFile(fileId) as ContextFile);
   }
 
   getFile(fileId: string): ContextFile | undefined {
@@ -181,15 +197,14 @@ export class ContextSession implements AsyncDisposable {
    * @param threshold relevance threshold for the similarity score, higher threshold means more similar chunks, default 0.7, good enough based on prior experiments
    * @returns list of similar chunks
    */
-  async matchFileChunks(
+  async matchFiles(
     content: string,
     topK: number = 5,
     signal?: AbortSignal,
-    threshold: number = 0.85
+    scopedThreshold: number = 0.85,
+    threshold: number = 0.5
   ): Promise<FileChunkSimilarity[]> {
-    const embedding = await this.client
-      .getEmbeddings([content], signal)
-      .then(r => r?.[0]?.embedding);
+    const embedding = await this.client.getEmbedding(content, signal);
     if (!embedding) return [];
 
     const [context, workspace] = await Promise.all([
@@ -197,7 +212,7 @@ export class ContextSession implements AsyncDisposable {
         embedding,
         this.id,
         topK * 2,
-        threshold
+        scopedThreshold
       ),
       this.models.copilotWorkspace.matchFileEmbedding(
         this.workspaceId,
@@ -206,10 +221,21 @@ export class ContextSession implements AsyncDisposable {
         threshold
       ),
     ]);
+    const files = new Map(this.files.map(f => [f.id, f]));
 
     return this.client.reRank(
       content,
-      [...context, ...workspace],
+      [
+        ...context
+          .filter(f => files.has(f.fileId))
+          .map(c => {
+            const { blobId, name, mimeType } = files.get(
+              c.fileId
+            ) as Required<ContextFile>;
+            return { ...c, blobId, name, mimeType };
+          }),
+        ...workspace,
+      ],
       topK,
       signal
     );
@@ -223,16 +249,14 @@ export class ContextSession implements AsyncDisposable {
    * @param threshold relevance threshold for the similarity score, higher threshold means more similar chunks, default 0.7, good enough based on prior experiments
    * @returns list of similar chunks
    */
-  async matchWorkspaceChunks(
+  async matchWorkspaceDocs(
     content: string,
     topK: number = 5,
     signal?: AbortSignal,
-    scopedThreshold: number = 0.5,
-    threshold: number = 0.85
+    scopedThreshold: number = 0.85,
+    threshold: number = 0.5
   ) {
-    const embedding = await this.client
-      .getEmbeddings([content], signal)
-      .then(r => r?.[0]?.embedding);
+    const embedding = await this.client.getEmbedding(content, signal);
     if (!embedding) return [];
 
     const docIds = this.docIds;

packages/backend/server/src/plugins/copilot/context/types.ts

@@ -128,6 +128,11 @@ export abstract class EmbeddingClient {
       .slice(0, topK);
   }
 
+  async getEmbedding(query: string, signal?: AbortSignal) {
+    const embedding = await this.getEmbeddings([query], signal);
+    return embedding?.[0]?.embedding;
+  }
+
   abstract getEmbeddings(
     input: string[],
     signal?: AbortSignal

packages/backend/server/src/plugins/copilot/workspace/service.ts

@@ -60,6 +60,7 @@ export class CopilotWorkspaceService implements OnApplicationBootstrap {
     await this.storage.put(userId, workspaceId, blobId, buffer);
     const file = await this.models.copilotWorkspace.addFile(workspaceId, {
       fileName,
+      blobId,
       mimeType: content.mimetype,
       size: buffer.length,
     });

packages/backend/server/src/plugins/copilot/workspace/types.ts

@@ -55,6 +55,9 @@ export class CopilotWorkspaceFileType implements CopilotWorkspaceFile {
   @Field(() => String)
   fileId!: string;
 
+  @Field(() => String)
+  blobId!: string;
+
   @Field(() => String)
   fileName!: string;
 

packages/backend/server/src/schema.gql

@@ -69,10 +69,13 @@ type ContextMatchedDocChunk {
 }
 
 type ContextMatchedFileChunk {
+  blobId: String!
   chunk: SafeInt!
   content: String!
   distance: Float
   fileId: String!
+  mimeType: String!
+  name: String!
 }
 
 type ContextWorkspaceEmbeddingStatus {
@@ -107,10 +110,10 @@ type CopilotContext {
 
   """list files in context"""
   files: [CopilotContextFile!]!
-  id: ID!
+  id: ID
 
   """match file in context"""
-  matchFiles(content: String!, limit: SafeInt, threshold: Float): [ContextMatchedFileChunk!]!
+  matchFiles(content: String!, limit: SafeInt, scopedThreshold: Float, threshold: Float): [ContextMatchedFileChunk!]!
 
   """match workspace docs"""
   matchWorkspaceDocs(content: String!, limit: SafeInt, scopedThreshold: Float, threshold: Float): [ContextMatchedDocChunk!]!
@@ -140,6 +143,7 @@ type CopilotContextFile {
   createdAt: SafeInt!
   error: String
   id: ID!
+  mimeType: String!
   name: String!
   status: ContextEmbedStatus!
 }
@@ -169,6 +173,12 @@ type CopilotFailedToMatchContextDataType {
   message: String!
 }
 
+type CopilotFailedToMatchGlobalContextDataType {
+  content: String!
+  message: String!
+  workspaceId: String!
+}
+
 type CopilotFailedToModifyContextDataType {
   contextId: String!
   message: String!
@@ -280,6 +290,7 @@ type CopilotWorkspaceConfig {
 }
 
 type CopilotWorkspaceFile {
+  blobId: String!
   createdAt: DateTime!
   fileId: String!
   fileName: String!
@@ -464,7 +475,7 @@ type EditorType {
   name: String!
 }
 
-union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CopilotContextFileNotSupportedDataType | CopilotDocNotFoundDataType | CopilotFailedToAddWorkspaceFileEmbeddingDataType | CopilotFailedToMatchContextDataType | CopilotFailedToModifyContextDataType | CopilotInvalidContextDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidLicenseToActivateDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NoMoreSeatDataType | NotInSpaceDataType | QueryTooLongDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType
+union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CopilotContextFileNotSupportedDataType | CopilotDocNotFoundDataType | CopilotFailedToAddWorkspaceFileEmbeddingDataType | CopilotFailedToMatchContextDataType | CopilotFailedToMatchGlobalContextDataType | CopilotFailedToModifyContextDataType | CopilotInvalidContextDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidLicenseToActivateDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NoMoreSeatDataType | NotInSpaceDataType | QueryTooLongDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType
 
 enum ErrorNames {
   ACCESS_DENIED
@@ -491,6 +502,7 @@ enum ErrorNames {
   COPILOT_FAILED_TO_CREATE_MESSAGE
   COPILOT_FAILED_TO_GENERATE_TEXT
   COPILOT_FAILED_TO_MATCH_CONTEXT
+  COPILOT_FAILED_TO_MATCH_GLOBAL_CONTEXT
   COPILOT_FAILED_TO_MODIFY_CONTEXT
   COPILOT_INVALID_CONTEXT
   COPILOT_MESSAGE_NOT_FOUND