refactor(server): config system (#11081)

packages/backend/server/src/plugins/copilot/providers/factory.ts

@@ -0,0 +1,107 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+import { ServerFeature, ServerService } from '../../../core';
+import type { FalProvider } from './fal';
+import type { GeminiProvider } from './gemini';
+import type { OpenAIProvider } from './openai';
+import type { PerplexityProvider } from './perplexity';
+import type { CopilotProvider } from './provider';
+import {
+  CapabilityToCopilotProvider,
+  CopilotCapability,
+  CopilotProviderType,
+} from './types';
+
+type TypedProvider = {
+  [CopilotProviderType.Gemini]: GeminiProvider;
+  [CopilotProviderType.OpenAI]: OpenAIProvider;
+  [CopilotProviderType.Perplexity]: PerplexityProvider;
+  [CopilotProviderType.FAL]: FalProvider;
+};
+
+@Injectable()
+export class CopilotProviderFactory {
+  constructor(private readonly server: ServerService) {}
+
+  private readonly logger = new Logger(CopilotProviderFactory.name);
+
+  readonly #providers = new Map<CopilotProviderType, CopilotProvider>();
+
+  getProvider<P extends CopilotProviderType>(provider: P): TypedProvider[P] {
+    return this.#providers.get(provider) as TypedProvider[P];
+  }
+
+  async getProviderByCapability<C extends CopilotCapability>(
+    capability: C,
+    filter: {
+      model?: string;
+      prefer?: CopilotProviderType;
+    } = {}
+  ): Promise<CapabilityToCopilotProvider[C] | null> {
+    this.logger.debug(
+      `Resolving copilot provider for capability: ${capability}`
+    );
+    let candidate: CopilotProvider | null = null;
+    for (const [type, provider] of this.#providers.entries()) {
+      // we firstly match by capability
+      if (provider.capabilities.includes(capability)) {
+        // use the first match if no filter provided
+        if (!filter.model && !filter.prefer) {
+          candidate = provider;
+          this.logger.debug(`Copilot provider candidate found: ${type}`);
+          break;
+        }
+
+        if (
+          (!filter.model || (await provider.isModelAvailable(filter.model))) &&
+          (!filter.prefer || filter.prefer === type)
+        ) {
+          candidate = provider;
+          this.logger.debug(`Copilot provider candidate found: ${type}`);
+          break;
+        }
+      }
+    }
+
+    return candidate as CapabilityToCopilotProvider[C] | null;
+  }
+
+  async getProviderByModel<C extends CopilotCapability>(
+    model: string,
+    filter: {
+      prefer?: CopilotProviderType;
+    } = {}
+  ): Promise<CapabilityToCopilotProvider[C] | null> {
+    this.logger.debug(`Resolving copilot provider for model: ${model}`);
+
+    let candidate: CopilotProvider | null = null;
+    for (const [type, provider] of this.#providers.entries()) {
+      // we firstly match by model
+      if (await provider.isModelAvailable(model)) {
+        candidate = provider;
+        this.logger.debug(`Copilot provider candidate found: ${type}`);
+
+        // then we match by prefer filter
+        if (!filter.prefer || filter.prefer === type) {
+          candidate = provider;
+        }
+      }
+    }
+
+    return candidate as CapabilityToCopilotProvider[C] | null;
+  }
+
+  register(provider: CopilotProvider) {
+    this.#providers.set(provider.type, provider);
+    this.logger.log(`Copilot provider [${provider.type}] registered.`);
+    this.server.enableFeature(ServerFeature.Copilot);
+  }
+
+  unregister(provider: CopilotProvider) {
+    this.#providers.delete(provider.type);
+    this.logger.log(`Copilot provider [${provider.type}] unregistered.`);
+    if (this.#providers.size === 0) {
+      this.server.disableFeature(ServerFeature.Copilot);
+    }
+  }
+}

packages/backend/server/src/plugins/copilot/providers/fal.ts

@@ -1,9 +1,8 @@
-import assert from 'node:assert';
-
 import {
   config as falConfig,
   stream as falStream,
 } from '@fal-ai/serverless-client';
+import { Injectable } from '@nestjs/common';
 import { z, ZodType } from 'zod';
 
 import {
@@ -12,6 +11,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   CopilotCapability,
   CopilotChatOptions,
@@ -20,7 +20,7 @@ import {
   CopilotProviderType,
   CopilotTextToImageProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export type FalConfig = {
   apiKey: string;
@@ -71,17 +71,19 @@ type FalPrompt = {
   }[];
 };
 
+@Injectable()
 export class FalProvider
+  extends CopilotProvider<FalConfig>
   implements CopilotTextToImageProvider, CopilotImageToImageProvider
 {
-  static readonly type = CopilotProviderType.FAL;
-  static readonly capabilities = [
+  override type = CopilotProviderType.FAL;
+  override readonly capabilities = [
     CopilotCapability.TextToImage,
     CopilotCapability.ImageToImage,
     CopilotCapability.ImageToText,
   ];
 
-  readonly availableModels = [
+  override readonly models = [
     // text to image
     'fast-turbo-diffusion',
     // image to image
@@ -96,27 +98,15 @@ export class FalProvider
     'llava-next',
   ];
 
-  constructor(private readonly config: FalConfig) {
-    assert(FalProvider.assetsConfig(config));
+  override configured(): boolean {
+    return !!this.config.apiKey;
+  }
+
+  protected override setup() {
+    super.setup();
     falConfig({ credentials: this.config.apiKey });
   }
 
-  static assetsConfig(config: FalConfig) {
-    return !!config.apiKey;
-  }
-
-  get type(): CopilotProviderType {
-    return FalProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return FalProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    return this.availableModels.includes(model);
-  }
-
   private extractArray<T>(value: T | T[] | undefined): T[] {
     if (Array.isArray(value)) return value;
     return value ? [value] : [];
@@ -211,7 +201,7 @@ export class FalProvider
     model: string = 'llava-next',
     options: CopilotChatOptions = {}
   ): Promise<string> {
-    if (!this.availableModels.includes(model)) {
+    if (!(await this.isModelAvailable(model))) {
       throw new CopilotPromptInvalid(`Invalid model: ${model}`);
     }
 
@@ -269,7 +259,7 @@ export class FalProvider
 
   private async buildResponse(
     messages: PromptMessage[],
-    model: string = this.availableModels[0],
+    model: string = this.models[0],
     options: CopilotImageOptions = {}
   ) {
     // by default, image prompt assumes there is only one message
@@ -300,10 +290,10 @@ export class FalProvider
   // ====== image to image ======
   async generateImages(
     messages: PromptMessage[],
-    model: string = this.availableModels[0],
+    model: string = this.models[0],
     options: CopilotImageOptions = {}
   ): Promise<Array<string>> {
-    if (!this.availableModels.includes(model)) {
+    if (!(await this.isModelAvailable(model))) {
       throw new CopilotPromptInvalid(`Invalid model: ${model}`);
     }
 
@@ -333,7 +323,7 @@ export class FalProvider
 
   async *generateImagesStream(
     messages: PromptMessage[],
-    model: string = this.availableModels[0],
+    model: string = this.models[0],
     options: CopilotImageOptions = {}
   ): AsyncIterable<string> {
     try {

packages/backend/server/src/plugins/copilot/providers/gemini.ts

@@ -2,7 +2,6 @@ import {
   createGoogleGenerativeAI,
   type GoogleGenerativeAIProvider,
 } from '@ai-sdk/google';
-import { Logger } from '@nestjs/common';
 import {
   AISDKError,
   type CoreAssistantMessage,
@@ -19,6 +18,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   ChatMessageRole,
   CopilotCapability,
@@ -26,7 +26,7 @@ import {
   CopilotProviderType,
   CopilotTextToTextProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export const DEFAULT_DIMENSIONS = 256;
 
@@ -49,45 +49,38 @@ const FORMAT_INFER_MAP: Record<string, string> = {
   flv: 'video/flv',
 };
 
-export type GoogleConfig = {
+export type GeminiConfig = {
   apiKey: string;
   baseUrl?: string;
 };
 
 type ChatMessage = CoreUserMessage | CoreAssistantMessage;
 
-export class GoogleProvider implements CopilotTextToTextProvider {
-  static readonly type = CopilotProviderType.Google;
-  static readonly capabilities = [CopilotCapability.TextToText];
-
-  readonly availableModels = [
+export class GeminiProvider
+  extends CopilotProvider<GeminiConfig>
+  implements CopilotTextToTextProvider
+{
+  override readonly type = CopilotProviderType.Gemini;
+  override readonly capabilities = [CopilotCapability.TextToText];
+  override readonly models = [
     // text to text
     'gemini-2.0-flash-001',
     // embeddings
     'text-embedding-004',
   ];
 
-  private readonly logger = new Logger(GoogleProvider.name);
-  private readonly instance: GoogleGenerativeAIProvider;
+  #instance!: GoogleGenerativeAIProvider;
 
-  constructor(config: GoogleConfig) {
-    this.instance = createGoogleGenerativeAI(config);
+  override configured(): boolean {
+    return !!this.config.apiKey;
   }
 
-  static assetsConfig(config: GoogleConfig) {
-    return !!config?.apiKey;
-  }
-
-  get type(): CopilotProviderType {
-    return GoogleProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return GoogleProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    return this.availableModels.includes(model);
+  protected override setup() {
+    super.setup();
+    this.#instance = createGoogleGenerativeAI({
+      apiKey: this.config.apiKey,
+      baseURL: this.config.baseUrl,
+    });
   }
 
   private inferMimeType(url: string) {
@@ -239,7 +232,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
       const [system, msgs] = await this.chatToGPTMessage(messages);
 
       const { text } = await generateText({
-        model: this.instance(model, {
+        model: this.#instance(model, {
           audioTimestamp: Boolean(options.audioTimestamp),
           structuredOutputs: Boolean(options.jsonMode),
         }),
@@ -268,7 +261,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
       const [system, msgs] = await this.chatToGPTMessage(messages);
 
       const { textStream } = streamText({
-        model: this.instance(model),
+        model: this.#instance(model),
         system,
         messages: msgs,
         abortSignal: options.signal,

packages/backend/server/src/plugins/copilot/providers/index.ts

@@ -1,203 +1,18 @@
-import assert from 'node:assert';
+import { FalProvider } from './fal';
+import { GeminiProvider } from './gemini';
+import { OpenAIProvider } from './openai';
+import { PerplexityProvider } from './perplexity';
 
-import { Injectable, Logger } from '@nestjs/common';
-
-import { AFFiNEConfig, Config } from '../../../base';
-import { CopilotStartupConfigurations } from '../config';
-import {
-  CapabilityToCopilotProvider,
-  CopilotCapability,
-  CopilotProvider,
-  CopilotProviderType,
-} from '../types';
-
-type CopilotProviderConfig =
-  CopilotStartupConfigurations[keyof CopilotStartupConfigurations];
-
-interface CopilotProviderDefinition<C extends CopilotProviderConfig> {
-  // constructor signature
-  new (config: C): CopilotProvider;
-  // type of the provider
-  readonly type: CopilotProviderType;
-  // capabilities of the provider, like text to text, text to image, etc.
-  readonly capabilities: CopilotCapability[];
-  // asserts that the config is valid for this provider
-  assetsConfig(config: C): boolean;
-}
-
-// registered provider factory
-const COPILOT_PROVIDER = new Map<
-  CopilotProviderType,
-  (config: Config, logger: Logger) => CopilotProvider
->();
-
-// map of capabilities to providers
-const PROVIDER_CAPABILITY_MAP = new Map<
-  CopilotCapability,
-  CopilotProviderType[]
->();
-
-// config assertions for providers
-const ASSERT_CONFIG = new Map<
-  CopilotProviderType,
-  (config: AFFiNEConfig) => void
->();
-
-export function registerCopilotProvider<
-  C extends CopilotProviderConfig = CopilotProviderConfig,
->(provider: CopilotProviderDefinition<C>) {
-  const type = provider.type;
-
-  const factory = (config: Config, logger: Logger) => {
-    const providerConfig = config.plugins.copilot?.[type];
-    if (!provider.assetsConfig(providerConfig as C)) {
-      throw new Error(
-        `Invalid configuration for copilot provider ${type}: ${JSON.stringify(providerConfig)}`
-      );
-    }
-    const instance = new provider(providerConfig as C);
-    logger.debug(
-      `Copilot provider ${type} registered, capabilities: ${provider.capabilities.join(', ')}`
-    );
-
-    return instance;
-  };
-  // register the provider
-  COPILOT_PROVIDER.set(type, factory);
-  // register the provider capabilities
-  for (const capability of provider.capabilities) {
-    const providers = PROVIDER_CAPABILITY_MAP.get(capability) || [];
-    if (!providers.includes(type)) {
-      providers.push(type);
-    }
-    PROVIDER_CAPABILITY_MAP.set(capability, providers);
-  }
-  // register the provider config assertion
-  ASSERT_CONFIG.set(type, (config: AFFiNEConfig) => {
-    assert(config.plugins.copilot);
-    const providerConfig = config.plugins.copilot[type];
-    if (!providerConfig) return false;
-    return provider.assetsConfig(providerConfig as C);
-  });
-}
-
-export function unregisterCopilotProvider(type: CopilotProviderType) {
-  COPILOT_PROVIDER.delete(type);
-  ASSERT_CONFIG.delete(type);
-  for (const providers of PROVIDER_CAPABILITY_MAP.values()) {
-    const index = providers.indexOf(type);
-    if (index !== -1) {
-      providers.splice(index, 1);
-    }
-  }
-}
-
-/// Asserts that the config is valid for any registered providers
-export function assertProvidersConfigs(config: AFFiNEConfig) {
-  return Array.from(ASSERT_CONFIG.values()).some(assertConfig =>
-    assertConfig(config)
-  );
-}
-
-@Injectable()
-export class CopilotProviderService {
-  private readonly logger = new Logger(CopilotProviderService.name);
-  constructor(private readonly config: Config) {}
-
-  private readonly cachedProviders = new Map<
-    CopilotProviderType,
-    CopilotProvider
-  >();
-
-  private create(provider: CopilotProviderType): CopilotProvider {
-    assert(this.config.plugins.copilot);
-    const providerFactory = COPILOT_PROVIDER.get(provider);
-
-    if (!providerFactory) {
-      throw new Error(`Unknown copilot provider type: ${provider}`);
-    }
-
-    return providerFactory(this.config, this.logger);
-  }
-
-  getProvider(provider: CopilotProviderType): CopilotProvider | null {
-    try {
-      if (!this.cachedProviders.has(provider)) {
-        this.cachedProviders.set(provider, this.create(provider));
-      }
-      return this.cachedProviders.get(provider) as CopilotProvider;
-    } catch {
-      // skip if the provider is not available
-      return null;
-    }
-  }
-
-  async getProviderByCapability<C extends CopilotCapability>(
-    capability: C,
-    model?: string,
-    prefer?: CopilotProviderType
-  ): Promise<CapabilityToCopilotProvider[C] | null> {
-    const providers = PROVIDER_CAPABILITY_MAP.get(capability);
-    if (Array.isArray(providers) && providers.length) {
-      let selectedProvider: CopilotProviderType | undefined = prefer;
-      let currentIndex = -1;
-
-      if (!selectedProvider) {
-        currentIndex = 0;
-        selectedProvider = providers[currentIndex];
-      }
-
-      while (selectedProvider) {
-        // find first provider that supports the capability and model
-        if (providers.includes(selectedProvider)) {
-          const provider = this.getProvider(selectedProvider);
-
-          if (provider && provider.getCapabilities().includes(capability)) {
-            if (model) {
-              if (await provider.isModelAvailable(model)) {
-                return provider as CapabilityToCopilotProvider[C];
-              }
-            } else {
-              return provider as CapabilityToCopilotProvider[C];
-            }
-          }
-        }
-        currentIndex += 1;
-        selectedProvider = providers[currentIndex];
-      }
-    }
-    return null;
-  }
-
-  async getProviderByModel<C extends CopilotCapability>(
-    model: string,
-    prefer?: CopilotProviderType
-  ): Promise<CapabilityToCopilotProvider[C] | null> {
-    const providers = Array.from(COPILOT_PROVIDER.keys());
-    if (providers.length) {
-      let selectedProvider: CopilotProviderType | undefined = prefer;
-      let currentIndex = -1;
-
-      if (!selectedProvider) {
-        currentIndex = 0;
-        selectedProvider = providers[currentIndex];
-      }
-
-      while (selectedProvider) {
-        const provider = this.getProvider(selectedProvider);
-
-        if (provider && (await provider.isModelAvailable(model))) {
-          return provider as CapabilityToCopilotProvider[C];
-        }
-
-        currentIndex += 1;
-        selectedProvider = providers[currentIndex];
-      }
-    }
-    return null;
-  }
-}
+export const CopilotProviders = [
+  OpenAIProvider,
+  FalProvider,
+  GeminiProvider,
+  PerplexityProvider,
+];
 
+export { CopilotProviderFactory } from './factory';
 export { FalProvider } from './fal';
+export { GeminiProvider } from './gemini';
 export { OpenAIProvider } from './openai';
 export { PerplexityProvider } from './perplexity';
+export * from './types';

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -1,4 +1,3 @@
-import { Logger } from '@nestjs/common';
 import { APIError, BadRequestError, ClientOptions, OpenAI } from 'openai';
 
 import {
@@ -7,6 +6,7 @@ import {
   metrics,
   UserFriendlyError,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   ChatMessageRole,
   CopilotCapability,
@@ -19,28 +19,31 @@ import {
   CopilotTextToImageProvider,
   CopilotTextToTextProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export const DEFAULT_DIMENSIONS = 256;
 
 const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
 
+export type OpenAIConfig = ClientOptions;
+
 export class OpenAIProvider
+  extends CopilotProvider<OpenAIConfig>
   implements
     CopilotTextToTextProvider,
     CopilotTextToEmbeddingProvider,
     CopilotTextToImageProvider,
     CopilotImageToTextProvider
 {
-  static readonly type = CopilotProviderType.OpenAI;
-  static readonly capabilities = [
+  readonly type = CopilotProviderType.OpenAI;
+  readonly capabilities = [
     CopilotCapability.TextToText,
     CopilotCapability.TextToEmbedding,
     CopilotCapability.TextToImage,
     CopilotCapability.ImageToText,
   ];
 
-  readonly availableModels = [
+  readonly models = [
     // text to text
     'gpt-4o',
     'gpt-4o-2024-08-06',
@@ -59,41 +62,32 @@ export class OpenAIProvider
     'dall-e-3',
   ];
 
-  private readonly logger = new Logger(OpenAIProvider.type);
-  private readonly instance: OpenAI;
+  #existsModels: string[] = [];
+  #instance!: OpenAI;
 
-  private existsModels: string[] | undefined;
-
-  constructor(config: ClientOptions) {
-    this.instance = new OpenAI(config);
+  override configured(): boolean {
+    return !!this.config.apiKey;
   }
 
-  static assetsConfig(config: ClientOptions) {
-    return !!config?.apiKey;
+  protected override setup() {
+    super.setup();
+    this.#instance = new OpenAI(this.config);
   }
 
-  get type(): CopilotProviderType {
-    return OpenAIProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return OpenAIProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    const knownModels = this.availableModels.includes(model);
+  override async isModelAvailable(model: string): Promise<boolean> {
+    const knownModels = this.models.includes(model);
     if (knownModels) return true;
 
-    if (!this.existsModels) {
+    if (!this.#existsModels) {
       try {
-        this.existsModels = await this.instance.models
+        this.#existsModels = await this.#instance.models
           .list()
           .then(({ data }) => data.map(m => m.id));
       } catch (e: any) {
         this.logger.error('Failed to fetch online model list', e.stack);
       }
     }
-    return !!this.existsModels?.includes(model);
+    return !!this.#existsModels?.includes(model);
   }
 
   protected chatToGPTMessage(
@@ -226,7 +220,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
-      const result = await this.instance.chat.completions.create(
+      const result = await this.#instance.chat.completions.create(
         {
           messages: this.chatToGPTMessage(messages),
           model: model,
@@ -257,7 +251,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model });
-      const result = await this.instance.chat.completions.create(
+      const result = await this.#instance.chat.completions.create(
         {
           stream: true,
           messages: this.chatToGPTMessage(messages),
@@ -307,7 +301,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('generate_embedding_calls').add(1, { model });
-      const result = await this.instance.embeddings.create({
+      const result = await this.#instance.embeddings.create({
         model: model,
         input: messages,
         dimensions: options.dimensions || DEFAULT_DIMENSIONS,
@@ -333,7 +327,7 @@ export class OpenAIProvider
 
     try {
       metrics.ai.counter('generate_images_calls').add(1, { model });
-      const result = await this.instance.images.generate(
+      const result = await this.#instance.images.generate(
         {
           prompt,
           model,

packages/backend/server/src/plugins/copilot/providers/perplexity.ts

@@ -1,5 +1,3 @@
-import assert from 'node:assert';
-
 import { EventSourceParserStream } from 'eventsource-parser/stream';
 import { z } from 'zod';
 
@@ -8,13 +6,14 @@ import {
   CopilotProviderSideError,
   metrics,
 } from '../../../base';
+import { CopilotProvider } from './provider';
 import {
   CopilotCapability,
   CopilotChatOptions,
   CopilotProviderType,
   CopilotTextToTextProvider,
   PromptMessage,
-} from '../types';
+} from './types';
 
 export type PerplexityConfig = {
   apiKey: string;
@@ -164,36 +163,21 @@ export class CitationParser {
   }
 }
 
-export class PerplexityProvider implements CopilotTextToTextProvider {
-  static readonly type = CopilotProviderType.Perplexity;
-
-  static readonly capabilities = [CopilotCapability.TextToText];
-
-  static assetsConfig(config: PerplexityConfig) {
-    return !!config.apiKey;
-  }
-
-  constructor(private readonly config: PerplexityConfig) {
-    assert(PerplexityProvider.assetsConfig(config));
-  }
-
-  readonly availableModels = [
+export class PerplexityProvider
+  extends CopilotProvider<PerplexityConfig>
+  implements CopilotTextToTextProvider
+{
+  readonly type = CopilotProviderType.Perplexity;
+  readonly capabilities = [CopilotCapability.TextToText];
+  readonly models = [
     'sonar',
     'sonar-pro',
     'sonar-reasoning',
     'sonar-reasoning-pro',
   ];
 
-  get type(): CopilotProviderType {
-    return PerplexityProvider.type;
-  }
-
-  getCapabilities(): CopilotCapability[] {
-    return PerplexityProvider.capabilities;
-  }
-
-  async isModelAvailable(model: string): Promise<boolean> {
-    return this.availableModels.includes(model);
+  override configured(): boolean {
+    return !!this.config.apiKey;
   }
 
   async generateText(

packages/backend/server/src/plugins/copilot/providers/provider.ts

@@ -0,0 +1,45 @@
+import { Inject, Injectable, Logger } from '@nestjs/common';
+
+import { Config, OnEvent } from '../../../base';
+import { CopilotProviderFactory } from './factory';
+import { CopilotCapability, CopilotProviderType } from './types';
+
+@Injectable()
+export abstract class CopilotProvider<C = any> {
+  protected readonly logger = new Logger(this.constructor.name);
+  abstract readonly type: CopilotProviderType;
+  abstract readonly capabilities: CopilotCapability[];
+  abstract readonly models: string[];
+  abstract configured(): boolean;
+
+  @Inject() protected readonly AFFiNEConfig!: Config;
+  @Inject() protected readonly factory!: CopilotProviderFactory;
+
+  get config(): C {
+    return this.AFFiNEConfig.copilot.providers[this.type] as C;
+  }
+
+  isModelAvailable(model: string): Promise<boolean> | boolean {
+    return this.models.includes(model);
+  }
+
+  @OnEvent('config.init')
+  async onConfigInit() {
+    this.setup();
+  }
+
+  @OnEvent('config.changed')
+  async onConfigChanged(event: Events['config.changed']) {
+    if ('copilot' in event.updates) {
+      this.setup();
+    }
+  }
+
+  protected setup() {
+    if (this.configured()) {
+      this.factory.register(this);
+    } else {
+      this.factory.unregister(this);
+    }
+  }
+}

packages/backend/server/src/plugins/copilot/providers/types.ts

@@ -0,0 +1,170 @@
+import { AiPromptRole } from '@prisma/client';
+import { z } from 'zod';
+
+import { type CopilotProvider } from './provider';
+
+export enum CopilotProviderType {
+  FAL = 'fal',
+  Gemini = 'gemini',
+  OpenAI = 'openai',
+  Perplexity = 'perplexity',
+}
+
+export enum CopilotCapability {
+  TextToText = 'text-to-text',
+  TextToEmbedding = 'text-to-embedding',
+  TextToImage = 'text-to-image',
+  ImageToImage = 'image-to-image',
+  ImageToText = 'image-to-text',
+}
+
+export const PromptConfigStrictSchema = z.object({
+  // openai
+  jsonMode: z.boolean().nullable().optional(),
+  frequencyPenalty: z.number().nullable().optional(),
+  presencePenalty: z.number().nullable().optional(),
+  temperature: z.number().nullable().optional(),
+  topP: z.number().nullable().optional(),
+  maxTokens: z.number().nullable().optional(),
+  // fal
+  modelName: z.string().nullable().optional(),
+  loras: z
+    .array(
+      z.object({ path: z.string(), scale: z.number().nullable().optional() })
+    )
+    .nullable()
+    .optional(),
+  // google
+  audioTimestamp: z.boolean().nullable().optional(),
+});
+
+export const PromptConfigSchema =
+  PromptConfigStrictSchema.nullable().optional();
+
+export type PromptConfig = z.infer<typeof PromptConfigSchema>;
+
+export const ChatMessageRole = Object.values(AiPromptRole) as [
+  'system',
+  'assistant',
+  'user',
+];
+
+export const PureMessageSchema = z.object({
+  content: z.string(),
+  attachments: z.array(z.string()).optional().nullable(),
+  params: z.record(z.any()).optional().nullable(),
+});
+
+export const PromptMessageSchema = PureMessageSchema.extend({
+  role: z.enum(ChatMessageRole),
+}).strict();
+export type PromptMessage = z.infer<typeof PromptMessageSchema>;
+export type PromptParams = NonNullable<PromptMessage['params']>;
+
+const CopilotProviderOptionsSchema = z.object({
+  signal: z.instanceof(AbortSignal).optional(),
+  user: z.string().optional(),
+});
+
+const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
+  PromptConfigStrictSchema
+).optional();
+
+export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
+
+const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({
+  dimensions: z.number(),
+}).optional();
+
+export type CopilotEmbeddingOptions = z.infer<
+  typeof CopilotEmbeddingOptionsSchema
+>;
+
+const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
+  PromptConfigStrictSchema
+)
+  .extend({
+    seed: z.number().optional(),
+  })
+  .optional();
+
+export type CopilotImageOptions = z.infer<typeof CopilotImageOptionsSchema>;
+
+export interface CopilotTextToTextProvider extends CopilotProvider {
+  generateText(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotChatOptions
+  ): Promise<string>;
+  generateTextStream(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotChatOptions
+  ): AsyncIterable<string>;
+}
+
+export interface CopilotTextToEmbeddingProvider extends CopilotProvider {
+  generateEmbedding(
+    messages: string[] | string,
+    model: string,
+    options?: CopilotEmbeddingOptions
+  ): Promise<number[][]>;
+}
+
+export interface CopilotTextToImageProvider extends CopilotProvider {
+  generateImages(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotImageOptions
+  ): Promise<Array<string>>;
+  generateImagesStream(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotImageOptions
+  ): AsyncIterable<string>;
+}
+
+export interface CopilotImageToTextProvider extends CopilotProvider {
+  generateText(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotChatOptions
+  ): Promise<string>;
+  generateTextStream(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotChatOptions
+  ): AsyncIterable<string>;
+}
+
+export interface CopilotImageToImageProvider extends CopilotProvider {
+  generateImages(
+    messages: PromptMessage[],
+    model: string,
+    options?: CopilotImageOptions
+  ): Promise<Array<string>>;
+  generateImagesStream(
+    messages: PromptMessage[],
+    model?: string,
+    options?: CopilotImageOptions
+  ): AsyncIterable<string>;
+}
+
+export type CapabilityToCopilotProvider = {
+  [CopilotCapability.TextToText]: CopilotTextToTextProvider;
+  [CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
+  [CopilotCapability.TextToImage]: CopilotTextToImageProvider;
+  [CopilotCapability.ImageToText]: CopilotImageToTextProvider;
+  [CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
+};
+
+export type CopilotTextProvider =
+  | CopilotTextToTextProvider
+  | CopilotImageToTextProvider;
+export type CopilotImageProvider =
+  | CopilotTextToImageProvider
+  | CopilotImageToImageProvider;
+export type CopilotAllProvider =
+  | CopilotTextProvider
+  | CopilotImageProvider
+  | CopilotTextToEmbeddingProvider;