feat(server): refactor copilot (#14892)

#### PR Dependency Tree


* **PR #14892** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

packages/backend/server/src/plugins/copilot/resolver.ts

@@ -1,5 +1,3 @@
-import { createHash } from 'node:crypto';
-
 import { BadRequestException, NotFoundException } from '@nestjs/common';
 import {
   Args,
@@ -24,37 +22,29 @@ import {
   CopilotProviderSideError,
   CopilotSessionNotFound,
   type FileUpload,
-  ImageFormatNotSupported,
   paginate,
   Paginated,
   PaginationInput,
   RequestMutex,
-  sniffMime,
   Throttle,
   TooManyRequest,
   UserFriendlyError,
 } from '../../base';
 import { CurrentUser } from '../../core/auth';
-import { Admin } from '../../core/common';
 import { DocReader } from '../../core/doc';
-import {
-  AccessController,
-  DocAction,
-  WorkspacePolicyService,
-} from '../../core/permission';
+import { AccessController, DocAction } from '../../core/permission';
 import { UserType } from '../../core/user';
 import type { ListSessionOptions, UpdateChatSession } from '../../models';
-import { processImage } from '../../native';
-import { CopilotCronJobs } from './cron';
+import { CompatHistoryProjector } from './compat/history-projector';
+import { ConversationInboxService } from './conversation/inbox';
 import { PromptService } from './prompt/service';
 import { CopilotProviderFactory } from './providers/factory';
-import type { PromptMessage, StreamObject } from './providers/types';
+import { ModelOutputType, type StreamObject } from './providers/types';
+import { CapabilityRuntime } from './runtime/capability-runtime';
 import { ChatSessionService } from './session';
-import { CopilotStorage } from './storage';
 import { type ChatHistory, type ChatMessage, SubmittedMessage } from './types';
 
 export const COPILOT_LOCKER = 'copilot';
-const COPILOT_IMAGE_MAX_EDGE = 1536;
 
 // ================== Input Types ==================
 
@@ -382,12 +372,13 @@ export class CopilotResolver {
   constructor(
     private readonly ac: AccessController,
     private readonly mutex: RequestMutex,
-    private readonly policy: WorkspacePolicyService,
     private readonly prompt: PromptService,
     private readonly chatSession: ChatSessionService,
-    private readonly storage: CopilotStorage,
+    private readonly historyProjector: CompatHistoryProjector,
+    private readonly inbox: ConversationInboxService,
     private readonly docReader: DocReader,
-    private readonly providerFactory: CopilotProviderFactory
+    private readonly providerFactory: CopilotProviderFactory,
+    private readonly runtime: CapabilityRuntime
   ) {}
 
   @ResolveField(() => CopilotQuotaType, {
@@ -436,33 +427,36 @@ export class CopilotResolver {
     if (!prompt) {
       throw new NotFoundException('Prompt not found');
     }
-    const convertModels = (ids: string[]) => {
-      return ids
-        .map(id => ({ id, name: this.modelNames.get(id) }))
-        .filter(m => !!m.name) as CopilotModelType[];
+    const convertModels = async (ids: string[]) => {
+      const models = await Promise.all(
+        ids.map(async id => {
+          const cachedName = this.modelNames.get(id);
+          if (cachedName) return { id, name: cachedName };
+
+          const resolved = await this.providerFactory.resolveProvider({
+            modelId: id,
+            outputType: ModelOutputType.Text,
+          });
+          const name = resolved?.provider.resolveModel(
+            resolved.modelId ?? id,
+            resolved.execution
+          )?.name;
+          if (name) {
+            this.modelNames.set(id, name);
+            return { id, name };
+          }
+          return null;
+        })
+      );
+
+      return models.filter(model => !!model) as CopilotModelType[];
     };
     const proModels = prompt.config?.proModels || [];
-    const missing = new Set(
-      [...prompt.optionalModels, ...proModels].filter(
-        id => !this.modelNames.has(id)
-      )
-    );
-    if (missing.size) {
-      for (const model of missing) {
-        if (this.modelNames.has(model)) continue;
-        const provider = await this.providerFactory.getProviderByModel(model);
-        if (provider?.configured()) {
-          for (const m of provider.models) {
-            if (m.name) this.modelNames.set(m.id, m.name);
-          }
-        }
-      }
-    }
 
     return {
       defaultModel: prompt.model,
-      optionalModels: convertModels(prompt.optionalModels),
-      proModels: convertModels(proModels),
+      optionalModels: await convertModels(prompt.optionalModels),
+      proModels: await convertModels(proModels),
     };
   }
 
@@ -476,11 +470,20 @@ export class CopilotResolver {
     @Args('sessionId') sessionId: string
   ): Promise<CopilotSessionType> {
     await this.assertPermission(user, copilot);
-    const session = await this.chatSession.getSessionInfo(sessionId);
-    if (!session) {
+    const state = await this.chatSession.getMetaState(sessionId);
+    if (!state) {
       throw new NotFoundException('Session not found');
     }
-    return this.transformToSessionType(session);
+
+    const projected = this.historyProjector.projectSession(state, {
+      requestUserId: user.id,
+      skipVisibilityFilter: true,
+    });
+    if (!projected) {
+      throw new NotFoundException('Session not found');
+    }
+
+    return this.transformToSessionType(projected);
   }
 
   @ResolveField(() => [CopilotSessionType], {
@@ -503,12 +506,19 @@ export class CopilotResolver {
       Object.assign({}, copilot, { docId: maybeDocId })
     );
 
-    const sessions = await this.chatSession.list(
-      Object.assign({}, options, appendOptions),
-      false
-    );
+    const sessions = (
+      await this.chatSession.listMetaStates(
+        Object.assign({}, options, appendOptions)
+      )
+    )
+      .map(state =>
+        this.historyProjector.projectSession(state, {
+          requestUserId: user.id,
+        })
+      )
+      .filter((history): history is Omit<ChatHistory, 'messages'> => !!history);
     if (appendOptions.docId) {
-      type Session = ChatHistory & { docId: string };
+      type Session = Omit<ChatHistory, 'messages'> & { docId: string };
       const filtered = sessions.filter((s): s is Session => !!s.docId);
       const accessible = await this.ac
         .user(user.id)
@@ -537,10 +547,20 @@ export class CopilotResolver {
       await this.assertPermission(user, { workspaceId, docId }, 'Doc.Read');
     }
 
-    const histories = await this.chatSession.list(
-      Object.assign({}, options, { userId: user.id, workspaceId, docId }),
-      true
-    );
+    const histories = (
+      await this.chatSession.listStates(
+        Object.assign({}, options, { userId: user.id, workspaceId, docId })
+      )
+    )
+      .map(state =>
+        this.historyProjector.projectHistory(state, {
+          requestUserId: user.id,
+          withMessages: true,
+          withPrompt: options?.withPrompt,
+          action: options?.action,
+        })
+      )
+      .filter((history): history is ChatHistory => !!history);
 
     return histories.map(h => ({
       ...h,
@@ -574,16 +594,31 @@ export class CopilotResolver {
       { skip: pagination.offset, limit: pagination.first }
     );
     const totalCount = await this.chatSession.count(finalOptions);
-    const histories = await this.chatSession.list(
-      finalOptions,
-      !!options?.withMessages
-    );
+    const histories: ChatHistory[] = options?.withMessages
+      ? (await this.chatSession.listStates(finalOptions))
+          .map(state =>
+            this.historyProjector.projectHistory(state, {
+              requestUserId: user.id,
+              withMessages: true,
+              withPrompt: options?.withPrompt,
+              action: options?.action,
+            })
+          )
+          .filter((history): history is ChatHistory => !!history)
+      : (await this.chatSession.listMetaStates(finalOptions)).flatMap(state => {
+          const session = this.historyProjector.projectSession(state, {
+            requestUserId: user.id,
+          });
+          return session
+            ? [{ ...session, messages: [] as ChatHistory['messages'] }]
+            : [];
+        });
 
     return paginate(
       histories.map(h => ({
         ...h,
         // filter out empty messages
-        messages: h.messages?.filter(
+        messages: h.messages.filter(
           m => m.content || m.attachments?.length
         ) as ChatMessageType[],
       })),
@@ -639,7 +674,16 @@ export class CopilotResolver {
     options: CreateChatSessionInput
   ): Promise<CopilotHistoriesType> {
     const sessionId = await this.createCopilotSessionInternal(user, options);
-    const session = await this.chatSession.getSessionInfo(sessionId);
+    const state = await this.chatSession.getState(sessionId);
+    if (!state) {
+      throw new NotFoundException('Session not found');
+    }
+    const session = this.historyProjector.projectHistory(state, {
+      requestUserId: user.id,
+      withMessages: true,
+      withPrompt: false,
+      action: !!state.prompt.action,
+    });
     if (!session) {
       throw new NotFoundException('Session not found');
     }
@@ -768,61 +812,8 @@ export class CopilotResolver {
     if (!lock) {
       throw new TooManyRequest('Server is busy');
     }
-    const session = await this.chatSession.get(options.sessionId);
-    if (!session || session.config.userId !== user.id) {
-      throw new BadRequestException('Session not found');
-    }
-
-    const attachments: PromptMessage['attachments'] = options.attachments || [];
-    if (options.blob || options.blobs) {
-      const { workspaceId } = session.config;
-
-      const blobs = await Promise.all(
-        options.blob ? [options.blob] : options.blobs || []
-      );
-      delete options.blob;
-      delete options.blobs;
-
-      if (blobs.length) {
-        await this.policy.assertCanUploadBlob(user.id, workspaceId);
-      }
-
-      for (const blob of blobs) {
-        const uploaded = await this.storage.handleUpload(user.id, blob);
-        const detectedMime =
-          sniffMime(uploaded.buffer, blob.mimetype)?.toLowerCase() ||
-          blob.mimetype;
-        let attachmentBuffer = uploaded.buffer;
-        let attachmentMimeType = detectedMime;
-
-        if (detectedMime.startsWith('image/')) {
-          try {
-            attachmentBuffer = await processImage(
-              uploaded.buffer,
-              COPILOT_IMAGE_MAX_EDGE,
-              true
-            );
-            attachmentMimeType = 'image/webp';
-          } catch {
-            throw new ImageFormatNotSupported({ format: detectedMime });
-          }
-        }
-
-        const filename = createHash('sha256')
-          .update(attachmentBuffer)
-          .digest('base64url');
-        const attachment = await this.storage.put(
-          user.id,
-          workspaceId,
-          filename,
-          attachmentBuffer
-        );
-        attachments.push({ attachment, mimeType: attachmentMimeType });
-      }
-    }
-
     try {
-      return await this.chatSession.createMessage({ ...options, attachments });
+      return await this.inbox.createMessage(user.id, options);
     } catch (e: any) {
       throw new CopilotFailedToCreateMessage(e.message);
     }
@@ -886,15 +877,16 @@ export class CopilotResolver {
 
     const markdown = docContent.markdown.trim();
 
-    // Get LLM provider
-    const provider =
-      await this.providerFactory.getProviderByModel('morph-v3-large');
-    if (!provider) {
+    const resolved = await this.providerFactory.resolveProvider({
+      modelId: 'morph-v3-large',
+      outputType: ModelOutputType.Text,
+    });
+    if (!resolved) {
       throw new BadRequestException('No LLM provider available');
     }
 
     try {
-      return await provider.text(
+      return await this.runtime.text(
         { modelId: 'morph-v3-large' },
         [
           {
@@ -909,7 +901,7 @@ export class CopilotResolver {
         throw e;
       } else {
         throw new CopilotProviderSideError({
-          provider: provider.type,
+          provider: resolved.provider.type,
           kind: 'unexpected_response',
           message: e?.message || 'Unexpected apply response',
         });
@@ -917,9 +909,7 @@ export class CopilotResolver {
     }
   }
 
-  private transformToSessionType(
-    session: Omit<ChatHistory, 'messages'>
-  ): CopilotSessionType {
+  private transformToSessionType(session: Omit<ChatHistory, 'messages'>) {
     return { id: session.sessionId, ...session };
   }
 }
@@ -944,25 +934,3 @@ export class UserCopilotResolver {
     return { workspaceId: workspaceId || null };
   }
 }
-
-@Admin()
-@Resolver(() => String)
-export class PromptsManagementResolver {
-  constructor(private readonly cron: CopilotCronJobs) {}
-
-  @Mutation(() => Boolean, {
-    description: 'Trigger generate missing titles cron job',
-  })
-  async triggerGenerateTitleCron() {
-    await this.cron.triggerGenerateMissingTitles();
-    return true;
-  }
-
-  @Mutation(() => Boolean, {
-    description: 'Trigger cleanup of trashed doc embeddings',
-  })
-  async triggerCleanupTrashedDocEmbeddings() {
-    await this.cron.triggerCleanupTrashedDocEmbeddings();
-    return true;
-  }
-}