refactor(server): config system (#11081)

packages/backend/server/src/plugins/copilot/config.ts

@@ -1,30 +1,76 @@
-import type { ClientOptions as OpenAIClientOptions } from 'openai';
-
-import { defineStartupConfig, ModuleConfig } from '../../base/config';
-import { StorageConfig } from '../../base/storage/config';
+import {
+  defineModuleConfig,
+  StorageJSONSchema,
+  StorageProviderConfig,
+} from '../../base';
 import type { FalConfig } from './providers/fal';
-import { GoogleConfig } from './providers/google';
+import { GeminiConfig } from './providers/gemini';
+import { OpenAIConfig } from './providers/openai';
 import { PerplexityConfig } from './providers/perplexity';
 
-export interface CopilotStartupConfigurations {
-  openai?: OpenAIClientOptions;
-  fal?: FalConfig;
-  google?: GoogleConfig;
-  perplexity?: PerplexityConfig;
-  test?: never;
-  unsplashKey?: string;
-  storage: StorageConfig;
-}
-
-declare module '../config' {
-  interface PluginsConfig {
-    copilot: ModuleConfig<CopilotStartupConfigurations>;
+declare global {
+  interface AppConfigSchema {
+    copilot: {
+      enabled: boolean;
+      unsplash: ConfigItem<{
+        key: string;
+      }>;
+      storage: ConfigItem<StorageProviderConfig>;
+      providers: {
+        openai: ConfigItem<OpenAIConfig>;
+        fal: ConfigItem<FalConfig>;
+        gemini: ConfigItem<GeminiConfig>;
+        perplexity: ConfigItem<PerplexityConfig>;
+      };
+    };
   }
 }
 
-defineStartupConfig('plugins.copilot', {
+defineModuleConfig('copilot', {
+  enabled: {
+    desc: 'Whether to enable the copilot plugin.',
+    default: false,
+  },
+  'providers.openai': {
+    desc: 'The config for the openai provider.',
+    default: {
+      apiKey: '',
+    },
+    link: 'https://github.com/openai/openai-node',
+  },
+  'providers.fal': {
+    desc: 'The config for the fal provider.',
+    default: {
+      apiKey: '',
+    },
+  },
+  'providers.gemini': {
+    desc: 'The config for the gemini provider.',
+    default: {
+      apiKey: '',
+    },
+  },
+  'providers.perplexity': {
+    desc: 'The config for the perplexity provider.',
+    default: {
+      apiKey: '',
+    },
+  },
+  unsplash: {
+    desc: 'The config for the unsplash key.',
+    default: {
+      key: '',
+    },
+  },
   storage: {
-    provider: 'fs',
-    bucket: 'copilot',
+    desc: 'The config for the storage provider.',
+    default: {
+      provider: 'fs',
+      bucket: 'copilot',
+      config: {
+        path: '~/.affine/storage',
+      },
+    },
+    schema: StorageJSONSchema,
   },
 });

packages/backend/server/src/plugins/copilot/context/job.ts

@@ -1,4 +1,4 @@
-import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import OpenAI from 'openai';
 
 import {
@@ -37,12 +37,12 @@ declare global {
 }
 
 @Injectable()
-export class CopilotContextDocJob implements OnModuleInit {
+export class CopilotContextDocJob {
   private supportEmbedding = false;
-  private readonly client: EmbeddingClient | undefined;
+  private client: EmbeddingClient | undefined;
 
   constructor(
-    config: Config,
+    private readonly config: Config,
     private readonly doc: DocReader,
     private readonly event: EventBus,
     private readonly logger: AFFiNELogger,
@@ -51,15 +51,24 @@ export class CopilotContextDocJob implements OnModuleInit {
     private readonly storage: CopilotStorage
   ) {
     this.logger.setContext(CopilotContextDocJob.name);
-    const configure = config.plugins.copilot.openai;
-    if (configure) {
-      this.client = new OpenAIEmbeddingClient(new OpenAI(configure));
-    }
   }
 
-  async onModuleInit() {
+  @OnEvent('config.init')
+  async onConfigInit() {
+    await this.setup();
+  }
+
+  @OnEvent('config.changed')
+  async onConfigChanged() {
+    await this.setup();
+  }
+
+  private async setup() {
     this.supportEmbedding =
       await this.models.copilotContext.checkEmbeddingAvailable();
+    this.client = new OpenAIEmbeddingClient(
+      new OpenAI(this.config.copilot.providers.openai)
+    );
   }
 
   // public this client to allow overriding in tests

packages/backend/server/src/plugins/copilot/context/service.ts

@@ -1,4 +1,4 @@
-import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
 import OpenAI from 'openai';
 
 import {
@@ -22,7 +22,7 @@ import { EmbeddingClient } from './types';
 const CONTEXT_SESSION_KEY = 'context-session';
 
 @Injectable()
-export class CopilotContextService implements OnModuleInit {
+export class CopilotContextService implements OnApplicationBootstrap {
   private supportEmbedding = false;
   private readonly client: EmbeddingClient | undefined;
 
@@ -31,13 +31,13 @@ export class CopilotContextService implements OnModuleInit {
     private readonly cache: Cache,
     private readonly models: Models
   ) {
-    const configure = config.plugins.copilot.openai;
+    const configure = config.copilot.providers.openai;
     if (configure) {
       this.client = new OpenAIEmbeddingClient(new OpenAI(configure));
     }
   }
 
-  async onModuleInit() {
+  async onApplicationBootstrap() {
     const supportEmbedding =
       await this.models.copilotContext.checkEmbeddingAvailable();
     if (supportEmbedding) {

packages/backend/server/src/plugins/copilot/controller.ts

@@ -45,10 +45,14 @@ import {
   UnsplashIsNotConfigured,
 } from '../../base';
 import { CurrentUser, Public } from '../../core/auth';
-import { CopilotProviderService } from './providers';
+import {
+  CopilotCapability,
+  CopilotProviderFactory,
+  CopilotTextProvider,
+} from './providers';
 import { ChatSession, ChatSessionService } from './session';
 import { CopilotStorage } from './storage';
-import { ChatMessage, CopilotCapability, CopilotTextProvider } from './types';
+import { ChatMessage } from './types';
 import { CopilotWorkflowService, GraphExecutorState } from './workflow';
 
 export interface ChatEvent {
@@ -72,7 +76,7 @@ export class CopilotController implements BeforeApplicationShutdown {
   constructor(
     private readonly config: Config,
     private readonly chatSession: ChatSessionService,
-    private readonly provider: CopilotProviderService,
+    private readonly provider: CopilotProviderFactory,
     private readonly workflow: CopilotWorkflowService,
     private readonly storage: CopilotStorage
   ) {}
@@ -121,13 +125,13 @@ export class CopilotController implements BeforeApplicationShutdown {
     );
     let provider = await this.provider.getProviderByCapability(
       CopilotCapability.TextToText,
-      model
+      { model }
     );
     // fallback to image to text if text to text is not available
     if (!provider && hasAttachment) {
       provider = await this.provider.getProviderByCapability(
         CopilotCapability.ImageToText,
-        model
+        { model }
       );
     }
     if (!provider) {
@@ -478,7 +482,7 @@ export class CopilotController implements BeforeApplicationShutdown {
         hasAttachment
           ? CopilotCapability.ImageToImage
           : CopilotCapability.TextToImage,
-        model
+        { model }
       );
       if (!provider) {
         throw new NoCopilotProviderAvailable();
@@ -565,8 +569,8 @@ export class CopilotController implements BeforeApplicationShutdown {
     @Res() res: Response,
     @Query() params: Record<string, string>
   ) {
-    const { unsplashKey } = this.config.plugins.copilot || {};
-    if (!unsplashKey) {
+    const { key } = this.config.copilot.unsplash;
+    if (!key) {
       throw new UnsplashIsNotConfigured();
     }
 
@@ -574,7 +578,7 @@ export class CopilotController implements BeforeApplicationShutdown {
     const response = await fetch(
       `https://api.unsplash.com/search/photos?${query}`,
       {
-        headers: { Authorization: `Client-ID ${unsplashKey}` },
+        headers: { Authorization: `Client-ID ${key}` },
         signal: this.getSignal(req),
       }
     );

packages/backend/server/src/plugins/copilot/index.ts

@@ -1,11 +1,12 @@
 import './config';
 
-import { ServerFeature } from '../../core/config';
+import { Module } from '@nestjs/common';
+
+import { ServerConfigModule } from '../../core';
 import { DocStorageModule } from '../../core/doc';
 import { FeatureModule } from '../../core/features';
 import { PermissionModule } from '../../core/permission';
 import { QuotaModule } from '../../core/quota';
-import { Plugin } from '../registry';
 import {
   CopilotContextDocJob,
   CopilotContextResolver,
@@ -15,15 +16,7 @@ import {
 import { CopilotController } from './controller';
 import { ChatMessageCache } from './message';
 import { PromptService } from './prompt';
-import {
-  assertProvidersConfigs,
-  CopilotProviderService,
-  FalProvider,
-  OpenAIProvider,
-  PerplexityProvider,
-  registerCopilotProvider,
-} from './providers';
-import { GoogleProvider } from './providers/google';
+import { CopilotProviderFactory, CopilotProviders } from './providers';
 import {
   CopilotResolver,
   PromptsManagementResolver,
@@ -37,42 +30,39 @@ import {
 } from './transcript';
 import { CopilotWorkflowExecutors, CopilotWorkflowService } from './workflow';
 
-registerCopilotProvider(FalProvider);
-registerCopilotProvider(OpenAIProvider);
-registerCopilotProvider(GoogleProvider);
-registerCopilotProvider(PerplexityProvider);
-
-@Plugin({
-  name: 'copilot',
-  imports: [DocStorageModule, FeatureModule, QuotaModule, PermissionModule],
+@Module({
+  imports: [
+    DocStorageModule,
+    FeatureModule,
+    QuotaModule,
+    PermissionModule,
+    ServerConfigModule,
+  ],
   providers: [
+    // providers
+    ...CopilotProviders,
+    CopilotProviderFactory,
+    // services
     ChatSessionService,
     CopilotResolver,
     ChatMessageCache,
-    UserCopilotResolver,
     PromptService,
-    CopilotProviderService,
     CopilotStorage,
-    PromptsManagementResolver,
     // workflow
     CopilotWorkflowService,
     ...CopilotWorkflowExecutors,
     // context
-    CopilotContextRootResolver,
     CopilotContextResolver,
     CopilotContextService,
     CopilotContextDocJob,
     // transcription
     CopilotTranscriptionService,
     CopilotTranscriptionResolver,
+    // gql resolvers
+    UserCopilotResolver,
+    PromptsManagementResolver,
+    CopilotContextRootResolver,
   ],
   controllers: [CopilotController],
-  contributesTo: ServerFeature.Copilot,
-  if: config => {
-    if (config.flavor.graphql || config.flavor.doc) {
-      return assertProvidersConfigs(config);
-    }
-    return false;
-  },
 })
 export class CopilotModule {}

packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts

@@ -3,12 +3,8 @@ import { Logger } from '@nestjs/common';
 import { AiPrompt } from '@prisma/client';
 import Mustache from 'mustache';
 
-import {
-  getTokenEncoder,
-  PromptConfig,
-  PromptMessage,
-  PromptParams,
-} from '../types';
+import { PromptConfig, PromptMessage, PromptParams } from '../providers';
+import { getTokenEncoder } from '../types';
 
 // disable escaping
 Mustache.escape = (text: string) => text;

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -1,7 +1,7 @@
 import { Logger } from '@nestjs/common';
 import { AiPrompt, PrismaClient } from '@prisma/client';
 
-import { PromptConfig, PromptMessage } from '../types';
+import { PromptConfig, PromptMessage } from '../providers';
 
 type Prompt = Omit<
   AiPrompt,

packages/backend/server/src/plugins/copilot/prompt/service.ts

@@ -1,4 +1,4 @@
-import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
 import { PrismaClient } from '@prisma/client';
 
 import {
@@ -6,17 +6,17 @@ import {
   PromptConfigSchema,
   PromptMessage,
   PromptMessageSchema,
-} from '../types';
+} from '../providers';
 import { ChatPrompt } from './chat-prompt';
 import { refreshPrompts } from './prompts';
 
 @Injectable()
-export class PromptService implements OnModuleInit {
+export class PromptService implements OnApplicationBootstrap {
   private readonly cache = new Map<string, ChatPrompt>();
 
   constructor(private readonly db: PrismaClient) {}
 
-  async onModuleInit() {
+  async onApplicationBootstrap() {
     this.cache.clear();
     await refreshPrompts(this.db);
   }

packages/backend/server/src/plugins/copilot/providers/factory.ts

@@ -0,0 +1,107 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { ServerFeature, ServerService } from '../../../core';
+import type { FalProvider } from './fal';
+import type { GeminiProvider } from './gemini';
+import type { OpenAIProvider } from './openai';
+import type { PerplexityProvider } from './perplexity';
+import type { CopilotProvider } from './provider';
+import {
+  CapabilityToCopilotProvider,
+  CopilotCapability,
+  CopilotProviderType,
+} from './types';
+
+type TypedProvider = {
+  [CopilotProviderType.Gemini]: GeminiProvider;
+  [CopilotProviderType.OpenAI]: OpenAIProvider;
+  [CopilotProviderType.Perplexity]: PerplexityProvider;
+  [CopilotProviderType.FAL]: FalProvider;
+};
+
+@Injectable()
+export class CopilotProviderFactory {
+  constructor(private readonly server: ServerService) {}
+
+  private readonly logger = new Logger(CopilotProviderFactory.name);
+
+  readonly #providers = new Map<CopilotProviderType, CopilotProvider>();
+
+  getProvider<P extends CopilotProviderType>(provider: P): TypedProvider[P] {
+    return this.#providers.get(provider) as TypedProvider[P];
+  }
+
+  async getProviderByCapability<C extends CopilotCapability>(
+    capability: C,
+    filter: {
+      model?: string;
+      prefer?: CopilotProviderType;
+    } = {}
+  ): Promise<CapabilityToCopilotProvider[C] | null> {
+    this.logger.debug(
+      `Resolving copilot provider for capability: ${capability}`
+    );
+    let candidate: CopilotProvider | null = null;
+    for (const [type, provider] of this.#providers.entries()) {
+      // we firstly match by capability
+      if (provider.capabilities.includes(capability)) {
+        // use the first match if no filter provided
+        if (!filter.model && !filter.prefer) {
+          candidate = provider;
+          this.logger.debug(`Copilot provider candidate found: ${type}`);
+          break;
+        }
+
+        if (
+          (!filter.model || (await provider.isModelAvailable(filter.model))) &&
+          (!filter.prefer || filter.prefer === type)
+        ) {
+          candidate = provider;
+          this.logger.debug(`Copilot provider candidate found: ${type}`);
+          break;
+        }
+      }
+    }
+
+    return candidate as CapabilityToCopilotProvider[C] | null;
+  }
+
+  async getProviderByModel<C extends CopilotCapability>(
+    model: string,
+    filter: {
+      prefer?: CopilotProviderType;
+    } = {}
+  ): Promise<CapabilityToCopilotProvider[C] | null> {
+    this.logger.debug(`Resolving copilot provider for model: ${model}`);
+
+    let candidate: CopilotProvider | null = null;
+    for (const [type, provider] of this.#providers.entries()) {
+      // we firstly match by model
+      if (await provider.isModelAvailable(model)) {
+        candidate = provider;
+        this.logger.debug(`Copilot provider candidate found: ${type}`);
+
+        // then we match by prefer filter
+        if (!filter.prefer || filter.prefer === type) {
+          candidate = provider;
+        }
+      }
+    }
+
+    return candidate as CapabilityToCopilotProvider[C] | null;
+  }
+
+  register(provider: CopilotProvider) {
+    this.#providers.set(provider.type, provider);
+    this.logger.log(`Copilot provider [${provider.type}] registered.`);
+    this.server.enableFeature(ServerFeature.Copilot);
+  }
+
+  unregister(provider: CopilotProvider) {
+    this.#providers.delete(provider.type);
+    this.logger.log(`Copilot provider [${provider.type}] unregistered.`);
+    if (this.#providers.size === 0) {
+      this.server.disableFeature(ServerFeature.Copilot);
+    }
+  }
+}

packages/backend/server/src/plugins/copilot/providers/fal.ts

@@ -1,9 +1,8 @@
-import assert from 'node:assert';
-
 import {
   config as falConfig,
   stream as falStream,
 } from '@fal-ai/serverless-client';
+import { Injectable } from '@nestjs/common';
 import { z, ZodType } from 'zod';
 
 import {
@@ -12,6 +11,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   CopilotCapability,
   CopilotChatOptions,
@@ -20,7 +20,7 @@ import {
   CopilotProviderType,
   CopilotTextToImageProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export type FalConfig = {
   apiKey: string;
@@ -71,17 +71,19 @@ type FalPrompt = {
   }[];
 };
 
+@Injectable()
 export class FalProvider
+  extends CopilotProvider<FalConfig>
   implements CopilotTextToImageProvider, CopilotImageToImageProvider
 {
-  static readonly type = CopilotProviderType.FAL;
-  static readonly capabilities = [
+  override type = CopilotProviderType.FAL;
+  override readonly capabilities = [
     CopilotCapability.TextToImage,
     CopilotCapability.ImageToImage,
     CopilotCapability.ImageToText,
   ];
 
-  readonly availableModels = [
+  override readonly models = [
     // text to image
     'fast-turbo-diffusion',
     // image to image
@@ -96,27 +98,15 @@ export class FalProvider
     'llava-next',
   ];
 
-  constructor(private readonly config: FalConfig) {
-    assert(FalProvider.assetsConfig(config));
+  override configured(): boolean {
+    return !!this.config.apiKey;
+  }
+
+  protected override setup() {
+    super.setup();
     falConfig({ credentials: this.config.apiKey });
   }
 
-  static assetsConfig(config: FalConfig) {
-    return !!config.apiKey;
-  }
-
-  get type(): CopilotProviderType {
-    return FalProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return FalProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    return this.availableModels.includes(model);
-  }
-
   private extractArray<T>(value: T | T[] | undefined): T[] {
     if (Array.isArray(value)) return value;
     return value ? [value] : [];
@@ -211,7 +201,7 @@ export class FalProvider
     model: string = 'llava-next',
     options: CopilotChatOptions = {}
   ): Promise<string> {
-    if (!this.availableModels.includes(model)) {
+    if (!(await this.isModelAvailable(model))) {
       throw new CopilotPromptInvalid(`Invalid model: ${model}`);
     }
 
@@ -269,7 +259,7 @@ export class FalProvider
 
   private async buildResponse(
     messages: PromptMessage[],
-    model: string = this.availableModels[0],
+    model: string = this.models[0],
     options: CopilotImageOptions = {}
   ) {
     // by default, image prompt assumes there is only one message
@@ -300,10 +290,10 @@ export class FalProvider
   // ====== image to image ======
   async generateImages(
     messages: PromptMessage[],
-    model: string = this.availableModels[0],
+    model: string = this.models[0],
     options: CopilotImageOptions = {}
   ): Promise<Array<string>> {
-    if (!this.availableModels.includes(model)) {
+    if (!(await this.isModelAvailable(model))) {
       throw new CopilotPromptInvalid(`Invalid model: ${model}`);
     }
 
@@ -333,7 +323,7 @@ export class FalProvider
 
   async *generateImagesStream(
     messages: PromptMessage[],
-    model: string = this.availableModels[0],
+    model: string = this.models[0],
     options: CopilotImageOptions = {}
   ): AsyncIterable<string> {
     try {

packages/backend/server/src/plugins/copilot/providers/gemini.ts

@@ -2,7 +2,6 @@ import {
   createGoogleGenerativeAI,
   type GoogleGenerativeAIProvider,
 } from '@ai-sdk/google';
-import { Logger } from '@nestjs/common';
 import {
   AISDKError,
   type CoreAssistantMessage,
@@ -19,6 +18,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   ChatMessageRole,
   CopilotCapability,
@@ -26,7 +26,7 @@ import {
   CopilotProviderType,
   CopilotTextToTextProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export const DEFAULT_DIMENSIONS = 256;
 
@@ -49,45 +49,38 @@ const FORMAT_INFER_MAP: Record<string, string> = {
   flv: 'video/flv',
 };
 
-export type GoogleConfig = {
+export type GeminiConfig = {
   apiKey: string;
   baseUrl?: string;
 };
 
 type ChatMessage = CoreUserMessage | CoreAssistantMessage;
 
-export class GoogleProvider implements CopilotTextToTextProvider {
-  static readonly type = CopilotProviderType.Google;
-  static readonly capabilities = [CopilotCapability.TextToText];
-
-  readonly availableModels = [
+export class GeminiProvider
+  extends CopilotProvider<GeminiConfig>
+  implements CopilotTextToTextProvider
+{
+  override readonly type = CopilotProviderType.Gemini;
+  override readonly capabilities = [CopilotCapability.TextToText];
+  override readonly models = [
     // text to text
     'gemini-2.0-flash-001',
     // embeddings
     'text-embedding-004',
   ];
 
-  private readonly logger = new Logger(GoogleProvider.name);
-  private readonly instance: GoogleGenerativeAIProvider;
+  #instance!: GoogleGenerativeAIProvider;
 
-  constructor(config: GoogleConfig) {
-    this.instance = createGoogleGenerativeAI(config);
+  override configured(): boolean {
+    return !!this.config.apiKey;
   }
 
-  static assetsConfig(config: GoogleConfig) {
-    return !!config?.apiKey;
-  }
-
-  get type(): CopilotProviderType {
-    return GoogleProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return GoogleProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    return this.availableModels.includes(model);
+  protected override setup() {
+    super.setup();
+    this.#instance = createGoogleGenerativeAI({
+      apiKey: this.config.apiKey,
+      baseURL: this.config.baseUrl,
+    });
   }
 
   private inferMimeType(url: string) {
@@ -239,7 +232,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
       const [system, msgs] = await this.chatToGPTMessage(messages);
 
       const { text } = await generateText({
-        model: this.instance(model, {
+        model: this.#instance(model, {
           audioTimestamp: Boolean(options.audioTimestamp),
           structuredOutputs: Boolean(options.jsonMode),
         }),
@@ -268,7 +261,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
       const [system, msgs] = await this.chatToGPTMessage(messages);
 
       const { textStream } = streamText({
-        model: this.instance(model),
+        model: this.#instance(model),
         system,
         messages: msgs,
         abortSignal: options.signal,

packages/backend/server/src/plugins/copilot/providers/index.ts

@@ -1,203 +1,18 @@
-import assert from 'node:assert';
+import { FalProvider } from './fal';
+import { GeminiProvider } from './gemini';
+import { OpenAIProvider } from './openai';
+import { PerplexityProvider } from './perplexity';
 
-import { Injectable, Logger } from '@nestjs/common';
-
-import { AFFiNEConfig, Config } from '../../../base';
-import { CopilotStartupConfigurations } from '../config';
-import {
-  CapabilityToCopilotProvider,
-  CopilotCapability,
-  CopilotProvider,
-  CopilotProviderType,
-} from '../types';
-
-type CopilotProviderConfig =
-  CopilotStartupConfigurations[keyof CopilotStartupConfigurations];
-
-interface CopilotProviderDefinition<C extends CopilotProviderConfig> {
-  // constructor signature
-  new (config: C): CopilotProvider;
-  // type of the provider
-  readonly type: CopilotProviderType;
-  // capabilities of the provider, like text to text, text to image, etc.
-  readonly capabilities: CopilotCapability[];
-  // asserts that the config is valid for this provider
-  assetsConfig(config: C): boolean;
-}
-
-// registered provider factory
-const COPILOT_PROVIDER = new Map<
-  CopilotProviderType,
-  (config: Config, logger: Logger) => CopilotProvider
->();
-
-// map of capabilities to providers
-const PROVIDER_CAPABILITY_MAP = new Map<
-  CopilotCapability,
-  CopilotProviderType[]
->();
-
-// config assertions for providers
-const ASSERT_CONFIG = new Map<
-  CopilotProviderType,
-  (config: AFFiNEConfig) => void
->();
-
-export function registerCopilotProvider<
-  C extends CopilotProviderConfig = CopilotProviderConfig,
->(provider: CopilotProviderDefinition<C>) {
-  const type = provider.type;
-
-  const factory = (config: Config, logger: Logger) => {
-    const providerConfig = config.plugins.copilot?.[type];
-    if (!provider.assetsConfig(providerConfig as C)) {
-      throw new Error(
-        `Invalid configuration for copilot provider ${type}: ${JSON.stringify(providerConfig)}`
-      );
-    }
-    const instance = new provider(providerConfig as C);
-    logger.debug(
-      `Copilot provider ${type} registered, capabilities: ${provider.capabilities.join(', ')}`
-    );
-
-    return instance;
-  };
-  // register the provider
-  COPILOT_PROVIDER.set(type, factory);
-  // register the provider capabilities
-  for (const capability of provider.capabilities) {
-    const providers = PROVIDER_CAPABILITY_MAP.get(capability) || [];
-    if (!providers.includes(type)) {
-      providers.push(type);
-    }
-    PROVIDER_CAPABILITY_MAP.set(capability, providers);
-  }
-  // register the provider config assertion
-  ASSERT_CONFIG.set(type, (config: AFFiNEConfig) => {
-    assert(config.plugins.copilot);
-    const providerConfig = config.plugins.copilot[type];
-    if (!providerConfig) return false;
-    return provider.assetsConfig(providerConfig as C);
-  });
-}
-
-export function unregisterCopilotProvider(type: CopilotProviderType) {
-  COPILOT_PROVIDER.delete(type);
-  ASSERT_CONFIG.delete(type);
-  for (const providers of PROVIDER_CAPABILITY_MAP.values()) {
-    const index = providers.indexOf(type);
-    if (index !== -1) {
-      providers.splice(index, 1);
-    }
-  }
-}
-
-/// Asserts that the config is valid for any registered providers
-export function assertProvidersConfigs(config: AFFiNEConfig) {
-  return Array.from(ASSERT_CONFIG.values()).some(assertConfig =>
-    assertConfig(config)
-  );
-}
-
-@Injectable()
-export class CopilotProviderService {
-  private readonly logger = new Logger(CopilotProviderService.name);
-  constructor(private readonly config: Config) {}
-
-  private readonly cachedProviders = new Map<
-    CopilotProviderType,
-    CopilotProvider
-  >();
-
-  private create(provider: CopilotProviderType): CopilotProvider {
-    assert(this.config.plugins.copilot);
-    const providerFactory = COPILOT_PROVIDER.get(provider);
-
-    if (!providerFactory) {
-      throw new Error(`Unknown copilot provider type: ${provider}`);
-    }
-
-    return providerFactory(this.config, this.logger);
-  }
-
-  getProvider(provider: CopilotProviderType): CopilotProvider | null {
-    try {
-      if (!this.cachedProviders.has(provider)) {
-        this.cachedProviders.set(provider, this.create(provider));
-      }
-      return this.cachedProviders.get(provider) as CopilotProvider;
-    } catch {
-      // skip if the provider is not available
-      return null;
-    }
-  }
-
-  async getProviderByCapability<C extends CopilotCapability>(
-    capability: C,
-    model?: string,
-    prefer?: CopilotProviderType
-  ): Promise<CapabilityToCopilotProvider[C] | null> {
-    const providers = PROVIDER_CAPABILITY_MAP.get(capability);
-    if (Array.isArray(providers) && providers.length) {
-      let selectedProvider: CopilotProviderType | undefined = prefer;
-      let currentIndex = -1;
-
-      if (!selectedProvider) {
-        currentIndex = 0;
-        selectedProvider = providers[currentIndex];
-      }
-
-      while (selectedProvider) {
-        // find first provider that supports the capability and model
-        if (providers.includes(selectedProvider)) {
-          const provider = this.getProvider(selectedProvider);
-
-          if (provider && provider.getCapabilities().includes(capability)) {
-            if (model) {
-              if (await provider.isModelAvailable(model)) {
-                return provider as CapabilityToCopilotProvider[C];
-              }
-            } else {
-              return provider as CapabilityToCopilotProvider[C];
-            }
-          }
-        }
-        currentIndex += 1;
-        selectedProvider = providers[currentIndex];
-      }
-    }
-    return null;
-  }
-
-  async getProviderByModel<C extends CopilotCapability>(
-    model: string,
-    prefer?: CopilotProviderType
-  ): Promise<CapabilityToCopilotProvider[C] | null> {
-    const providers = Array.from(COPILOT_PROVIDER.keys());
-    if (providers.length) {
-      let selectedProvider: CopilotProviderType | undefined = prefer;
-      let currentIndex = -1;
-
-      if (!selectedProvider) {
-        currentIndex = 0;
-        selectedProvider = providers[currentIndex];
-      }
-
-      while (selectedProvider) {
-        const provider = this.getProvider(selectedProvider);
-
-        if (provider && (await provider.isModelAvailable(model))) {
-          return provider as CapabilityToCopilotProvider[C];
-        }
-
-        currentIndex += 1;
-        selectedProvider = providers[currentIndex];
-      }
-    }
-    return null;
-  }
-}
+export const CopilotProviders = [
+  OpenAIProvider,
+  FalProvider,
+  GeminiProvider,
+  PerplexityProvider,
+];
 
+export { CopilotProviderFactory } from './factory';
 export { FalProvider } from './fal';
+export { GeminiProvider } from './gemini';
 export { OpenAIProvider } from './openai';
 export { PerplexityProvider } from './perplexity';
+export * from './types';

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -1,4 +1,3 @@
-import { Logger } from '@nestjs/common';
 import { APIError, BadRequestError, ClientOptions, OpenAI } from 'openai';
 
 import {
@@ -7,6 +6,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   ChatMessageRole,
   CopilotCapability,
@@ -19,28 +19,31 @@ import {
   CopilotTextToImageProvider,
   CopilotTextToTextProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export const DEFAULT_DIMENSIONS = 256;
 
 const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
 
+export type OpenAIConfig = ClientOptions;
+
 export class OpenAIProvider
+  extends CopilotProvider<OpenAIConfig>
   implements
     CopilotTextToTextProvider,
     CopilotTextToEmbeddingProvider,
     CopilotTextToImageProvider,
     CopilotImageToTextProvider
 {
-  static readonly type = CopilotProviderType.OpenAI;
-  static readonly capabilities = [
+  readonly type = CopilotProviderType.OpenAI;
+  readonly capabilities = [
     CopilotCapability.TextToText,
     CopilotCapability.TextToEmbedding,
     CopilotCapability.TextToImage,
     CopilotCapability.ImageToText,
   ];
 
-  readonly availableModels = [
+  readonly models = [
     // text to text
     'gpt-4o',
     'gpt-4o-2024-08-06',
@@ -59,41 +62,32 @@ export class OpenAIProvider
     'dall-e-3',
   ];
 
-  private readonly logger = new Logger(OpenAIProvider.type);
-  private readonly instance: OpenAI;
+  #existsModels: string[] = [];
+  #instance!: OpenAI;
 
-  private existsModels: string[] | undefined;
-
-  constructor(config: ClientOptions) {
-    this.instance = new OpenAI(config);
+  override configured(): boolean {
+    return !!this.config.apiKey;
   }
 
-  static assetsConfig(config: ClientOptions) {
-    return !!config?.apiKey;
+  protected override setup() {
+    super.setup();
+    this.#instance = new OpenAI(this.config);
   }
 
-  get type(): CopilotProviderType {
-    return OpenAIProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return OpenAIProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    const knownModels = this.availableModels.includes(model);
+  override async isModelAvailable(model: string): Promise<boolean> {
+    const knownModels = this.models.includes(model);
     if (knownModels) return true;
 
-    if (!this.existsModels) {
+    if (!this.#existsModels) {
       try {
-        this.existsModels = await this.instance.models
+        this.#existsModels = await this.#instance.models
           .list()
           .then(({ data }) => data.map(m => m.id));
       } catch (e: any) {
         this.logger.error('Failed to fetch online model list', e.stack);
       }
     }
-    return !!this.existsModels?.includes(model);
+    return !!this.#existsModels?.includes(model);
   }
 
   protected chatToGPTMessage(
@@ -226,7 +220,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
-      const result = await this.instance.chat.completions.create(
+      const result = await this.#instance.chat.completions.create(
         {
           messages: this.chatToGPTMessage(messages),
           model: model,
@@ -257,7 +251,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model });
-      const result = await this.instance.chat.completions.create(
+      const result = await this.#instance.chat.completions.create(
         {
           stream: true,
           messages: this.chatToGPTMessage(messages),
@@ -307,7 +301,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('generate_embedding_calls').add(1, { model });
-      const result = await this.instance.embeddings.create({
+      const result = await this.#instance.embeddings.create({
         model: model,
         input: messages,
         dimensions: options.dimensions || DEFAULT_DIMENSIONS,
@@ -333,7 +327,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('generate_images_calls').add(1, { model });
-      const result = await this.instance.images.generate(
+      const result = await this.#instance.images.generate(
         {
           prompt,
           model,

packages/backend/server/src/plugins/copilot/providers/perplexity.ts

@@ -1,5 +1,3 @@
-import assert from 'node:assert';
-
 import { EventSourceParserStream } from 'eventsource-parser/stream';
 import { z } from 'zod';
 
@@ -8,13 +6,14 @@ import {
   CopilotProviderSideError,
   metrics,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   CopilotCapability,
   CopilotChatOptions,
   CopilotProviderType,
   CopilotTextToTextProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export type PerplexityConfig = {
   apiKey: string;
@@ -164,36 +163,21 @@ export class CitationParser {
   }
 }
 
-export class PerplexityProvider implements CopilotTextToTextProvider {
-  static readonly type = CopilotProviderType.Perplexity;
-
-  static readonly capabilities = [CopilotCapability.TextToText];
-
-  static assetsConfig(config: PerplexityConfig) {
-    return !!config.apiKey;
-  }
-
-  constructor(private readonly config: PerplexityConfig) {
-    assert(PerplexityProvider.assetsConfig(config));
-  }
-
-  readonly availableModels = [
+export class PerplexityProvider
+  extends CopilotProvider<PerplexityConfig>
+  implements CopilotTextToTextProvider
+{
+  readonly type = CopilotProviderType.Perplexity;
+  readonly capabilities = [CopilotCapability.TextToText];
+  readonly models = [
     'sonar',
     'sonar-pro',
     'sonar-reasoning',
     'sonar-reasoning-pro',
   ];
 
-  get type(): CopilotProviderType {
-    return PerplexityProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return PerplexityProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    return this.availableModels.includes(model);
+  override configured(): boolean {
+    return !!this.config.apiKey;
   }
 
   async generateText(

packages/backend/server/src/plugins/copilot/providers/provider.ts

@@ -0,0 +1,45 @@
+import { Inject, Injectable, Logger } from '@nestjs/common';
+
+import { Config, OnEvent } from '../../../base';
+import { CopilotProviderFactory } from './factory';
+import { CopilotCapability, CopilotProviderType } from './types';
+
+@Injectable()
+export abstract class CopilotProvider<C = any> {
+  protected readonly logger = new Logger(this.constructor.name);
+  abstract readonly type: CopilotProviderType;
+  abstract readonly capabilities: CopilotCapability[];
+  abstract readonly models: string[];
+  abstract configured(): boolean;
+
+  @Inject() protected readonly AFFiNEConfig!: Config;
+  @Inject() protected readonly factory!: CopilotProviderFactory;
+
+  get config(): C {
+    return this.AFFiNEConfig.copilot.providers[this.type] as C;
+  }
+
+  isModelAvailable(model: string): Promise<boolean> | boolean {
+    return this.models.includes(model);
+  }
+
+  @OnEvent('config.init')
+  async onConfigInit() {
+    this.setup();
+  }
+
+  @OnEvent('config.changed')
+  async onConfigChanged(event: Events['config.changed']) {
+    if ('copilot' in event.updates) {
+      this.setup();
+    }
+  }
+
+  protected setup() {
+    if (this.configured()) {
+      this.factory.register(this);
+    } else {
+      this.factory.unregister(this);
+    }
+  }
+}

packages/backend/server/src/plugins/copilot/providers/types.ts

@@ -0,0 +1,170 @@
+import { AiPromptRole } from '@prisma/client';
+import { z } from 'zod';
+
+import { type CopilotProvider } from './provider';
+
+export enum CopilotProviderType {
+  FAL = 'fal',
+  Gemini = 'gemini',
+  OpenAI = 'openai',
+  Perplexity = 'perplexity',
+}
+
+export enum CopilotCapability {
+  TextToText = 'text-to-text',
+  TextToEmbedding = 'text-to-embedding',
+  TextToImage = 'text-to-image',
+  ImageToImage = 'image-to-image',
+  ImageToText = 'image-to-text',
+}
+
+export const PromptConfigStrictSchema = z.object({
+  // openai
+  jsonMode: z.boolean().nullable().optional(),
+  frequencyPenalty: z.number().nullable().optional(),
+  presencePenalty: z.number().nullable().optional(),
+  temperature: z.number().nullable().optional(),
+  topP: z.number().nullable().optional(),
+  maxTokens: z.number().nullable().optional(),
+  // fal
+  modelName: z.string().nullable().optional(),
+  loras: z
+    .array(
+      z.object({ path: z.string(), scale: z.number().nullable().optional() })
+    )
+    .nullable()
+    .optional(),
+  // google
+  audioTimestamp: z.boolean().nullable().optional(),
+});
+
+export const PromptConfigSchema =
+  PromptConfigStrictSchema.nullable().optional();
+
+export type PromptConfig = z.infer<typeof PromptConfigSchema>;
+
+export const ChatMessageRole = Object.values(AiPromptRole) as [
+  'system',
+  'assistant',
+  'user',
+];
+
+export const PureMessageSchema = z.object({
+  content: z.string(),
+  attachments: z.array(z.string()).optional().nullable(),
+  params: z.record(z.any()).optional().nullable(),
+});
+
+export const PromptMessageSchema = PureMessageSchema.extend({
+  role: z.enum(ChatMessageRole),
+}).strict();
+export type PromptMessage = z.infer<typeof PromptMessageSchema>;
+export type PromptParams = NonNullable<PromptMessage['params']>;
+
+const CopilotProviderOptionsSchema = z.object({
+  signal: z.instanceof(AbortSignal).optional(),
+  user: z.string().optional(),
+});
+
+const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
+  PromptConfigStrictSchema
+).optional();
+
+export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
+
+const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({
+  dimensions: z.number(),
+}).optional();
+
+export type CopilotEmbeddingOptions = z.infer<
+  typeof CopilotEmbeddingOptionsSchema
+>;
+
+const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
+  PromptConfigStrictSchema
+)
+  .extend({
+    seed: z.number().optional(),
+  })
+  .optional();
+
+export type CopilotImageOptions = z.infer<typeof CopilotImageOptionsSchema>;
+
+export interface CopilotTextToTextProvider extends CopilotProvider {
+  generateText(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotChatOptions
+  ): Promise<string>;
+  generateTextStream(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotChatOptions
+  ): AsyncIterable<string>;
+}
+
+export interface CopilotTextToEmbeddingProvider extends CopilotProvider {
+  generateEmbedding(
+    messages: string[] | string,
+    model: string,
+    options?: CopilotEmbeddingOptions
+  ): Promise<number[][]>;
+}
+
+export interface CopilotTextToImageProvider extends CopilotProvider {
+  generateImages(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotImageOptions
+  ): Promise<Array<string>>;
+  generateImagesStream(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotImageOptions
+  ): AsyncIterable<string>;
+}
+
+export interface CopilotImageToTextProvider extends CopilotProvider {
+  generateText(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotChatOptions
+  ): Promise<string>;
+  generateTextStream(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotChatOptions
+  ): AsyncIterable<string>;
+}
+
+export interface CopilotImageToImageProvider extends CopilotProvider {
+  generateImages(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotImageOptions
+  ): Promise<Array<string>>;
+  generateImagesStream(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotImageOptions
+  ): AsyncIterable<string>;
+}
+
+export type CapabilityToCopilotProvider = {
+  [CopilotCapability.TextToText]: CopilotTextToTextProvider;
+  [CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
+  [CopilotCapability.TextToImage]: CopilotTextToImageProvider;
+  [CopilotCapability.ImageToText]: CopilotImageToTextProvider;
+  [CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
+};
+
+export type CopilotTextProvider =
+  | CopilotTextToTextProvider
+  | CopilotImageToTextProvider;
+export type CopilotImageProvider =
+  | CopilotTextToImageProvider
+  | CopilotImageToImageProvider;
+export type CopilotAllProvider =
+  | CopilotTextProvider
+  | CopilotImageProvider
+  | CopilotTextToEmbeddingProvider;

packages/backend/server/src/plugins/copilot/session.ts

@@ -17,6 +17,7 @@ import { QuotaService } from '../../core/quota';
 import { Models } from '../../models';
 import { ChatMessageCache } from './message';
 import { PromptService } from './prompt';
+import { PromptMessage, PromptParams } from './providers';
 import {
   AvailableModel,
   ChatHistory,
@@ -28,8 +29,6 @@ import {
   ChatSessionState,
   getTokenEncoder,
   ListHistoriesOptions,
-  PromptMessage,
-  PromptParams,
   SubmittedMessage,
 } from './types';
 
@@ -533,15 +532,17 @@ export class ChatSessionService {
                 const ret = ChatMessageSchema.array().safeParse(messages);
                 if (ret.success) {
                   // render system prompt
-                  const preload = options?.withPrompt
-                    ? prompt
-                        .finish(ret.data[0]?.params || {}, id)
-                        .filter(({ role }) => role !== 'system')
-                    : [];
+                  const preload = (
+                    options?.withPrompt
+                      ? prompt
+                          .finish(ret.data[0]?.params || {}, id)
+                          .filter(({ role }) => role !== 'system')
+                      : []
+                  ) as ChatMessage[];
 
                   // `createdAt` is required for history sorting in frontend
                   // let's fake the creating time of prompt messages
-                  (preload as ChatMessage[]).forEach((msg, i) => {
+                  preload.forEach((msg, i) => {
                     msg.createdAt = new Date(
                       createdAt.getTime() - preload.length - i - 1
                     );

packages/backend/server/src/plugins/copilot/storage.ts

@@ -25,9 +25,7 @@ export class CopilotStorage {
     private readonly storageFactory: StorageProviderFactory,
     private readonly quota: QuotaService
   ) {
-    this.provider = this.storageFactory.create(
-      this.config.plugins.copilot.storage
-    );
+    this.provider = this.storageFactory.create(this.config.copilot.storage);
   }
 
   @CallMetric('ai', 'blob_put')
@@ -39,7 +37,7 @@ export class CopilotStorage {
   ) {
     const name = `${userId}/${workspaceId}/${key}`;
     await this.provider.put(name, blob);
-    if (this.config.node.dev || this.config.node.test) {
+    if (!env.prod) {
       // return image base64url for dev environment
       return `data:image/png;base64,${blob.toString('base64')}`;
     }

packages/backend/server/src/plugins/copilot/transcript/service.ts

@@ -11,13 +11,13 @@ import {
 } from '../../../base';
 import { Models } from '../../../models';
 import { PromptService } from '../prompt';
-import { CopilotProviderService } from '../providers';
-import { CopilotStorage } from '../storage';
 import {
   CopilotCapability,
+  CopilotProviderFactory,
   CopilotTextProvider,
   PromptMessage,
-} from '../types';
+} from '../providers';
+import { CopilotStorage } from '../storage';
 import {
   TranscriptionPayload,
   TranscriptionSchema,
@@ -38,7 +38,7 @@ export class CopilotTranscriptionService {
     private readonly job: JobQueue,
     private readonly storage: CopilotStorage,
     private readonly prompt: PromptService,
-    private readonly provider: CopilotProviderService
+    private readonly providerFactory: CopilotProviderFactory
   ) {}
 
   async submitTranscriptionJob(
@@ -123,9 +123,9 @@ export class CopilotTranscriptionService {
   }
 
   private async getProvider(model: string): Promise<CopilotTextProvider> {
-    let provider = await this.provider.getProviderByCapability(
+    let provider = await this.providerFactory.getProviderByCapability(
       CopilotCapability.TextToText,
-      model
+      { model }
     );
 
     if (!provider) {

packages/backend/server/src/plugins/copilot/types.ts

@@ -1,9 +1,9 @@
 import { type Tokenizer } from '@affine/server-native';
-import { AiPromptRole } from '@prisma/client';
 import { z } from 'zod';
 
 import { fromModelName } from '../../native';
 import type { ChatPrompt } from './prompt';
+import { PromptMessageSchema, PureMessageSchema } from './providers';
 
 export enum AvailableModels {
   // text to text
@@ -41,77 +41,30 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null {
 
 // ======== ChatMessage ========
 
-export const ChatMessageRole = Object.values(AiPromptRole) as [
-  'system',
-  'assistant',
-  'user',
-];
-
-const PureMessageSchema = z.object({
-  content: z.string(),
-  attachments: z.array(z.string()).optional().nullable(),
-  params: z.record(z.any()).optional().nullable(),
-});
-
-export const PromptMessageSchema = PureMessageSchema.extend({
-  role: z.enum(ChatMessageRole),
-}).strict();
-
-export type PromptMessage = z.infer<typeof PromptMessageSchema>;
-
-export type PromptParams = NonNullable<PromptMessage['params']>;
-
-export const PromptConfigStrictSchema = z.object({
-  // openai
-  jsonMode: z.boolean().nullable().optional(),
-  frequencyPenalty: z.number().nullable().optional(),
-  presencePenalty: z.number().nullable().optional(),
-  temperature: z.number().nullable().optional(),
-  topP: z.number().nullable().optional(),
-  maxTokens: z.number().nullable().optional(),
-  // fal
-  modelName: z.string().nullable().optional(),
-  loras: z
-    .array(
-      z.object({ path: z.string(), scale: z.number().nullable().optional() })
-    )
-    .nullable()
-    .optional(),
-  // google
-  audioTimestamp: z.boolean().nullable().optional(),
-});
-
-export const PromptConfigSchema =
-  PromptConfigStrictSchema.nullable().optional();
-
-export type PromptConfig = z.infer<typeof PromptConfigSchema>;
-
 export const ChatMessageSchema = PromptMessageSchema.extend({
   id: z.string().optional(),
   createdAt: z.date(),
 }).strict();
-
 export type ChatMessage = z.infer<typeof ChatMessageSchema>;
 
-export const SubmittedMessageSchema = PureMessageSchema.extend({
-  sessionId: z.string(),
-  content: z.string().optional(),
-}).strict();
-
-export type SubmittedMessage = z.infer<typeof SubmittedMessageSchema>;
-
 export const ChatHistorySchema = z
   .object({
     sessionId: z.string(),
     action: z.string().nullable(),
     tokens: z.number(),
-    messages: z.array(PromptMessageSchema.or(ChatMessageSchema)),
+    messages: z.array(ChatMessageSchema),
     createdAt: z.date(),
   })
   .strict();
 
 export type ChatHistory = z.infer<typeof ChatHistorySchema>;
 
+export const SubmittedMessageSchema = PureMessageSchema.extend({
+  sessionId: z.string(),
+  content: z.string().optional(),
+}).strict();
+export type SubmittedMessage = z.infer<typeof SubmittedMessageSchema>;
+
 // ======== Chat Session ========
 
 export interface ChatSessionOptions {
@@ -154,142 +107,9 @@ export type ListHistoriesOptions = {
   withPrompt: boolean | undefined;
 };
 
-// ======== Provider Interface ========
-
-export enum CopilotProviderType {
-  FAL = 'fal',
-  Google = 'google',
-  OpenAI = 'openai',
-  Perplexity = 'perplexity',
-  // only for test
-  Test = 'test',
-}
-
-export enum CopilotCapability {
-  TextToText = 'text-to-text',
-  TextToEmbedding = 'text-to-embedding',
-  TextToImage = 'text-to-image',
-  ImageToImage = 'image-to-image',
-  ImageToText = 'image-to-text',
-}
-
-const CopilotProviderOptionsSchema = z.object({
-  signal: z.instanceof(AbortSignal).optional(),
-  user: z.string().optional(),
-});
-
-const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
-  PromptConfigStrictSchema
-).optional();
-
-export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
-
-const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({
-  dimensions: z.number(),
-}).optional();
-
-export type CopilotEmbeddingOptions = z.infer<
-  typeof CopilotEmbeddingOptionsSchema
->;
-
-const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
-  PromptConfigStrictSchema
-)
-  .extend({
-    seed: z.number().optional(),
-  })
-  .optional();
-
-export type CopilotImageOptions = z.infer<typeof CopilotImageOptionsSchema>;
-
 export type CopilotContextFile = {
   id: string; // fileId
   created_at: number;
   // embedding status
   status: 'in_progress' | 'completed' | 'failed';
 };
-
-export interface CopilotProvider {
-  readonly type: CopilotProviderType;
-  getCapabilities(): CopilotCapability[];
-  isModelAvailable(model: string): Promise<boolean>;
-}
-
-export interface CopilotTextToTextProvider extends CopilotProvider {
-  generateText(
-    messages: PromptMessage[],
-    model?: string,
-    options?: CopilotChatOptions
-  ): Promise<string>;
-  generateTextStream(
-    messages: PromptMessage[],
-    model?: string,
-    options?: CopilotChatOptions
-  ): AsyncIterable<string>;
-}
-
-export interface CopilotTextToEmbeddingProvider extends CopilotProvider {
-  generateEmbedding(
-    messages: string[] | string,
-    model: string,
-    options?: CopilotEmbeddingOptions
-  ): Promise<number[][]>;
-}
-
-export interface CopilotTextToImageProvider extends CopilotProvider {
-  generateImages(
-    messages: PromptMessage[],
-    model: string,
-    options?: CopilotImageOptions
-  ): Promise<Array<string>>;
-  generateImagesStream(
-    messages: PromptMessage[],
-    model?: string,
-    options?: CopilotImageOptions
-  ): AsyncIterable<string>;
-}
-
-export interface CopilotImageToTextProvider extends CopilotProvider {
-  generateText(
-    messages: PromptMessage[],
-    model: string,
-    options?: CopilotChatOptions
-  ): Promise<string>;
-  generateTextStream(
-    messages: PromptMessage[],
-    model: string,
-    options?: CopilotChatOptions
-  ): AsyncIterable<string>;
-}
-
-export interface CopilotImageToImageProvider extends CopilotProvider {
-  generateImages(
-    messages: PromptMessage[],
-    model: string,
-    options?: CopilotImageOptions
-  ): Promise<Array<string>>;
-  generateImagesStream(
-    messages: PromptMessage[],
-    model?: string,
-    options?: CopilotImageOptions
-  ): AsyncIterable<string>;
-}
-
-export type CapabilityToCopilotProvider = {
-  [CopilotCapability.TextToText]: CopilotTextToTextProvider;
-  [CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
-  [CopilotCapability.TextToImage]: CopilotTextToImageProvider;
-  [CopilotCapability.ImageToText]: CopilotImageToTextProvider;
-  [CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
-};
-
-export type CopilotTextProvider =
-  | CopilotTextToTextProvider
-  | CopilotImageToTextProvider;
-export type CopilotImageProvider =
-  | CopilotTextToImageProvider
-  | CopilotImageToImageProvider;
-export type CopilotAllProvider =
-  | CopilotTextProvider
-  | CopilotImageProvider
-  | CopilotTextToEmbeddingProvider;

packages/backend/server/src/plugins/copilot/workflow/executor/chat-image.ts

@@ -1,8 +1,11 @@
 import { Injectable } from '@nestjs/common';
 
 import { ChatPrompt, PromptService } from '../../prompt';
-import { CopilotProviderService } from '../../providers';
-import { CopilotChatOptions, CopilotImageProvider } from '../../types';
+import {
+  CopilotChatOptions,
+  CopilotImageProvider,
+  CopilotProviderFactory,
+} from '../../providers';
 import { WorkflowNodeData, WorkflowNodeType } from '../types';
 import { NodeExecuteResult, NodeExecuteState, NodeExecutorType } from './types';
 import { AutoRegisteredWorkflowExecutor } from './utils';
@@ -11,7 +14,7 @@ import { AutoRegisteredWorkflowExecutor } from './utils';
 export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor {
   constructor(
     private readonly promptService: PromptService,
-    private readonly providerService: CopilotProviderService
+    private readonly providerFactory: CopilotProviderFactory
   ) {
     super();
   }
@@ -42,7 +45,7 @@ export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor {
         `Prompt ${data.promptName} not found when running workflow node ${data.name}`
       );
     }
-    const provider = await this.providerService.getProviderByModel(
+    const provider = await this.providerFactory.getProviderByModel(
       prompt.model
     );
     if (provider && 'generateImages' in provider) {

packages/backend/server/src/plugins/copilot/workflow/executor/chat-text.ts

@@ -1,8 +1,11 @@
 import { Injectable } from '@nestjs/common';
 
 import { ChatPrompt, PromptService } from '../../prompt';
-import { CopilotProviderService } from '../../providers';
-import { CopilotChatOptions, CopilotTextProvider } from '../../types';
+import {
+  CopilotChatOptions,
+  CopilotProviderFactory,
+  CopilotTextProvider,
+} from '../../providers';
 import { WorkflowNodeData, WorkflowNodeType } from '../types';
 import { NodeExecuteResult, NodeExecuteState, NodeExecutorType } from './types';
 import { AutoRegisteredWorkflowExecutor } from './utils';
@@ -11,7 +14,7 @@ import { AutoRegisteredWorkflowExecutor } from './utils';
 export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor {
   constructor(
     private readonly promptService: PromptService,
-    private readonly providerService: CopilotProviderService
+    private readonly providerFactory: CopilotProviderFactory
   ) {
     super();
   }
@@ -42,7 +45,7 @@ export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor {
         `Prompt ${data.promptName} not found when running workflow node ${data.name}`
       );
     }
-    const provider = await this.providerService.getProviderByModel(
+    const provider = await this.providerFactory.getProviderByModel(
       prompt.model
     );
     if (provider && 'generateText' in provider) {

packages/backend/server/src/plugins/copilot/workflow/executor/types.ts

@@ -1,4 +1,4 @@
-import { CopilotChatOptions } from '../../types';
+import { CopilotChatOptions } from '../../providers';
 import type { WorkflowNode } from '../node';
 import { WorkflowNodeData, WorkflowParams } from '../types';
 

packages/backend/server/src/plugins/copilot/workflow/node.ts

@@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url';
 import { Logger } from '@nestjs/common';
 import Piscina from 'piscina';
 
-import { CopilotChatOptions } from '../types';
+import { CopilotChatOptions } from '../providers';
 import type { NodeExecuteResult, NodeExecutor } from './executor';
 import { getWorkflowExecutor, NodeExecuteState } from './executor';
 import type {

packages/backend/server/src/plugins/copilot/workflow/service.ts

@@ -1,6 +1,6 @@
 import { Injectable, Logger } from '@nestjs/common';
 
-import { CopilotChatOptions } from '../types';
+import { CopilotChatOptions } from '../providers';
 import { WorkflowGraphList } from './graph';
 import { WorkflowNode } from './node';
 import type { WorkflowGraph, WorkflowGraphInstances } from './types';

packages/backend/server/src/plugins/copilot/workflow/workflow.ts

@@ -1,6 +1,6 @@
 import { Logger } from '@nestjs/common';
 
-import { CopilotChatOptions } from '../types';
+import { CopilotChatOptions } from '../providers';
 import { NodeExecuteState } from './executor';
 import { WorkflowNode } from './node';
 import type { WorkflowGraphInstances, WorkflowNodeState } from './types';