mirror of
https://github.com/toeverything/AFFiNE.git
synced 2026-07-12 23:56:36 +08:00
refactor(server): config system (#11081)
This commit is contained in:
@@ -1,18 +1,12 @@
|
||||
import {
|
||||
defineRuntimeConfig,
|
||||
defineStartupConfig,
|
||||
ModuleConfig,
|
||||
} from '../../base/config';
|
||||
import { defineModuleConfig } from '../../base';
|
||||
import { CaptchaConfig } from './types';
|
||||
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
captcha: ModuleConfig<
|
||||
CaptchaConfig,
|
||||
{
|
||||
enable: boolean;
|
||||
}
|
||||
>;
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
captcha: {
|
||||
enabled: boolean;
|
||||
config: ConfigItem<CaptchaConfig>;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,18 +16,20 @@ declare module '../../base/guard' {
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.captcha', {
|
||||
turnstile: {
|
||||
secret: '',
|
||||
},
|
||||
challenge: {
|
||||
bits: 20,
|
||||
},
|
||||
});
|
||||
|
||||
defineRuntimeConfig('plugins.captcha', {
|
||||
enable: {
|
||||
defineModuleConfig('captcha', {
|
||||
enabled: {
|
||||
desc: 'Check captcha challenge when user authenticating the app.',
|
||||
default: false,
|
||||
},
|
||||
config: {
|
||||
desc: 'The config for the captcha plugin.',
|
||||
default: {
|
||||
turnstile: {
|
||||
secret: '',
|
||||
},
|
||||
challenge: {
|
||||
bits: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -6,9 +6,9 @@ import type {
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
Config,
|
||||
getRequestResponseFromContext,
|
||||
GuardProvider,
|
||||
Runtime,
|
||||
} from '../../base';
|
||||
import { CaptchaService } from './service';
|
||||
|
||||
@@ -20,14 +20,14 @@ export class CaptchaGuardProvider
|
||||
name = 'captcha' as const;
|
||||
|
||||
constructor(
|
||||
private readonly captcha: CaptchaService,
|
||||
private readonly runtime: Runtime
|
||||
private readonly config: Config,
|
||||
private readonly captcha: CaptchaService
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async canActivate(context: ExecutionContext) {
|
||||
if (!(await this.runtime.fetch('plugins.captcha/enable'))) {
|
||||
if (!this.config.captcha.enabled) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
import './config';
|
||||
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ServerConfigModule } from '../../core';
|
||||
import { AuthModule } from '../../core/auth';
|
||||
import { ServerFeature } from '../../core/config';
|
||||
import { Plugin } from '../registry';
|
||||
import { CaptchaController } from './controller';
|
||||
import { CaptchaGuardProvider } from './guard';
|
||||
import { CaptchaService } from './service';
|
||||
|
||||
@Plugin({
|
||||
name: 'captcha',
|
||||
imports: [AuthModule],
|
||||
@Module({
|
||||
imports: [AuthModule, ServerConfigModule],
|
||||
providers: [CaptchaService, CaptchaGuardProvider],
|
||||
controllers: [CaptchaController],
|
||||
contributesTo: ServerFeature.Captcha,
|
||||
requires: ['plugins.captcha.turnstile.secret'],
|
||||
})
|
||||
export class CaptchaModule {}
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import assert from 'node:assert';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
@@ -6,12 +5,10 @@ import type { Request } from 'express';
|
||||
import { nanoid } from 'nanoid';
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
CaptchaVerificationFailed,
|
||||
Config,
|
||||
verifyChallengeResponse,
|
||||
} from '../../base';
|
||||
import { CaptchaVerificationFailed, Config, OnEvent } from '../../base';
|
||||
import { ServerFeature, ServerService } from '../../core';
|
||||
import { Models, TokenType } from '../../models';
|
||||
import { verifyChallengeResponse } from '../../native';
|
||||
import { CaptchaConfig } from './types';
|
||||
|
||||
const validator = z
|
||||
@@ -26,10 +23,22 @@ export class CaptchaService {
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly models: Models
|
||||
private readonly models: Models,
|
||||
private readonly server: ServerService
|
||||
) {
|
||||
assert(config.plugins.captcha);
|
||||
this.captcha = config.plugins.captcha;
|
||||
this.captcha = config.captcha.config;
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
onConfigInit() {
|
||||
this.setup();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if ('captcha' in event.updates) {
|
||||
this.setup();
|
||||
}
|
||||
}
|
||||
|
||||
private async verifyCaptchaToken(token: any, ip: string) {
|
||||
@@ -52,7 +61,7 @@ export class CaptchaService {
|
||||
return (
|
||||
!!outcome.success &&
|
||||
// skip hostname check in dev mode
|
||||
(this.config.node.dev || outcome.hostname === this.config.server.host)
|
||||
(env.dev || outcome.hostname === this.config.server.host)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -119,4 +128,12 @@ export class CaptchaService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private setup() {
|
||||
if (this.config.captcha.enabled) {
|
||||
this.server.enableFeature(ServerFeature.Captcha);
|
||||
} else {
|
||||
this.server.disableFeature(ServerFeature.Captcha);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import { ModuleStartupConfigDescriptions } from '../base/config/types';
|
||||
|
||||
export interface PluginsConfig {}
|
||||
export type AvailablePlugins = keyof PluginsConfig;
|
||||
|
||||
declare module '../base/config' {
|
||||
interface AppConfig {
|
||||
plugins: PluginsConfig;
|
||||
}
|
||||
|
||||
interface AppPluginsConfig {
|
||||
use<Plugin extends AvailablePlugins>(
|
||||
plugin: Plugin,
|
||||
config?: DeepPartial<
|
||||
ModuleStartupConfigDescriptions<PluginsConfig[Plugin]>
|
||||
>
|
||||
): void;
|
||||
plugins: {
|
||||
/**
|
||||
* @deprecated use `AFFiNE.use` instead
|
||||
*/
|
||||
use<Plugin extends AvailablePlugins>(
|
||||
plugin: Plugin,
|
||||
config?: DeepPartial<
|
||||
ModuleStartupConfigDescriptions<PluginsConfig[Plugin]>
|
||||
>
|
||||
): void;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,76 @@
|
||||
import type { ClientOptions as OpenAIClientOptions } from 'openai';
|
||||
|
||||
import { defineStartupConfig, ModuleConfig } from '../../base/config';
|
||||
import { StorageConfig } from '../../base/storage/config';
|
||||
import {
|
||||
defineModuleConfig,
|
||||
StorageJSONSchema,
|
||||
StorageProviderConfig,
|
||||
} from '../../base';
|
||||
import type { FalConfig } from './providers/fal';
|
||||
import { GoogleConfig } from './providers/google';
|
||||
import { GeminiConfig } from './providers/gemini';
|
||||
import { OpenAIConfig } from './providers/openai';
|
||||
import { PerplexityConfig } from './providers/perplexity';
|
||||
|
||||
export interface CopilotStartupConfigurations {
|
||||
openai?: OpenAIClientOptions;
|
||||
fal?: FalConfig;
|
||||
google?: GoogleConfig;
|
||||
perplexity?: PerplexityConfig;
|
||||
test?: never;
|
||||
unsplashKey?: string;
|
||||
storage: StorageConfig;
|
||||
}
|
||||
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
copilot: ModuleConfig<CopilotStartupConfigurations>;
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
copilot: {
|
||||
enabled: boolean;
|
||||
unsplash: ConfigItem<{
|
||||
key: string;
|
||||
}>;
|
||||
storage: ConfigItem<StorageProviderConfig>;
|
||||
providers: {
|
||||
openai: ConfigItem<OpenAIConfig>;
|
||||
fal: ConfigItem<FalConfig>;
|
||||
gemini: ConfigItem<GeminiConfig>;
|
||||
perplexity: ConfigItem<PerplexityConfig>;
|
||||
};
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.copilot', {
|
||||
defineModuleConfig('copilot', {
|
||||
enabled: {
|
||||
desc: 'Whether to enable the copilot plugin.',
|
||||
default: false,
|
||||
},
|
||||
'providers.openai': {
|
||||
desc: 'The config for the openai provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
},
|
||||
link: 'https://github.com/openai/openai-node',
|
||||
},
|
||||
'providers.fal': {
|
||||
desc: 'The config for the fal provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
},
|
||||
},
|
||||
'providers.gemini': {
|
||||
desc: 'The config for the gemini provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
},
|
||||
},
|
||||
'providers.perplexity': {
|
||||
desc: 'The config for the perplexity provider.',
|
||||
default: {
|
||||
apiKey: '',
|
||||
},
|
||||
},
|
||||
unsplash: {
|
||||
desc: 'The config for the unsplash key.',
|
||||
default: {
|
||||
key: '',
|
||||
},
|
||||
},
|
||||
storage: {
|
||||
provider: 'fs',
|
||||
bucket: 'copilot',
|
||||
desc: 'The config for the storage provider.',
|
||||
default: {
|
||||
provider: 'fs',
|
||||
bucket: 'copilot',
|
||||
config: {
|
||||
path: '~/.affine/storage',
|
||||
},
|
||||
},
|
||||
schema: StorageJSONSchema,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import OpenAI from 'openai';
|
||||
|
||||
import {
|
||||
@@ -37,12 +37,12 @@ declare global {
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class CopilotContextDocJob implements OnModuleInit {
|
||||
export class CopilotContextDocJob {
|
||||
private supportEmbedding = false;
|
||||
private readonly client: EmbeddingClient | undefined;
|
||||
private client: EmbeddingClient | undefined;
|
||||
|
||||
constructor(
|
||||
config: Config,
|
||||
private readonly config: Config,
|
||||
private readonly doc: DocReader,
|
||||
private readonly event: EventBus,
|
||||
private readonly logger: AFFiNELogger,
|
||||
@@ -51,15 +51,24 @@ export class CopilotContextDocJob implements OnModuleInit {
|
||||
private readonly storage: CopilotStorage
|
||||
) {
|
||||
this.logger.setContext(CopilotContextDocJob.name);
|
||||
const configure = config.plugins.copilot.openai;
|
||||
if (configure) {
|
||||
this.client = new OpenAIEmbeddingClient(new OpenAI(configure));
|
||||
}
|
||||
}
|
||||
|
||||
async onModuleInit() {
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
await this.setup();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged() {
|
||||
await this.setup();
|
||||
}
|
||||
|
||||
private async setup() {
|
||||
this.supportEmbedding =
|
||||
await this.models.copilotContext.checkEmbeddingAvailable();
|
||||
this.client = new OpenAIEmbeddingClient(
|
||||
new OpenAI(this.config.copilot.providers.openai)
|
||||
);
|
||||
}
|
||||
|
||||
// public this client to allow overriding in tests
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import OpenAI from 'openai';
|
||||
|
||||
import {
|
||||
@@ -22,7 +22,7 @@ import { EmbeddingClient } from './types';
|
||||
const CONTEXT_SESSION_KEY = 'context-session';
|
||||
|
||||
@Injectable()
|
||||
export class CopilotContextService implements OnModuleInit {
|
||||
export class CopilotContextService implements OnApplicationBootstrap {
|
||||
private supportEmbedding = false;
|
||||
private readonly client: EmbeddingClient | undefined;
|
||||
|
||||
@@ -31,13 +31,13 @@ export class CopilotContextService implements OnModuleInit {
|
||||
private readonly cache: Cache,
|
||||
private readonly models: Models
|
||||
) {
|
||||
const configure = config.plugins.copilot.openai;
|
||||
const configure = config.copilot.providers.openai;
|
||||
if (configure) {
|
||||
this.client = new OpenAIEmbeddingClient(new OpenAI(configure));
|
||||
}
|
||||
}
|
||||
|
||||
async onModuleInit() {
|
||||
async onApplicationBootstrap() {
|
||||
const supportEmbedding =
|
||||
await this.models.copilotContext.checkEmbeddingAvailable();
|
||||
if (supportEmbedding) {
|
||||
|
||||
@@ -45,10 +45,14 @@ import {
|
||||
UnsplashIsNotConfigured,
|
||||
} from '../../base';
|
||||
import { CurrentUser, Public } from '../../core/auth';
|
||||
import { CopilotProviderService } from './providers';
|
||||
import {
|
||||
CopilotCapability,
|
||||
CopilotProviderFactory,
|
||||
CopilotTextProvider,
|
||||
} from './providers';
|
||||
import { ChatSession, ChatSessionService } from './session';
|
||||
import { CopilotStorage } from './storage';
|
||||
import { ChatMessage, CopilotCapability, CopilotTextProvider } from './types';
|
||||
import { ChatMessage } from './types';
|
||||
import { CopilotWorkflowService, GraphExecutorState } from './workflow';
|
||||
|
||||
export interface ChatEvent {
|
||||
@@ -72,7 +76,7 @@ export class CopilotController implements BeforeApplicationShutdown {
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly chatSession: ChatSessionService,
|
||||
private readonly provider: CopilotProviderService,
|
||||
private readonly provider: CopilotProviderFactory,
|
||||
private readonly workflow: CopilotWorkflowService,
|
||||
private readonly storage: CopilotStorage
|
||||
) {}
|
||||
@@ -121,13 +125,13 @@ export class CopilotController implements BeforeApplicationShutdown {
|
||||
);
|
||||
let provider = await this.provider.getProviderByCapability(
|
||||
CopilotCapability.TextToText,
|
||||
model
|
||||
{ model }
|
||||
);
|
||||
// fallback to image to text if text to text is not available
|
||||
if (!provider && hasAttachment) {
|
||||
provider = await this.provider.getProviderByCapability(
|
||||
CopilotCapability.ImageToText,
|
||||
model
|
||||
{ model }
|
||||
);
|
||||
}
|
||||
if (!provider) {
|
||||
@@ -478,7 +482,7 @@ export class CopilotController implements BeforeApplicationShutdown {
|
||||
hasAttachment
|
||||
? CopilotCapability.ImageToImage
|
||||
: CopilotCapability.TextToImage,
|
||||
model
|
||||
{ model }
|
||||
);
|
||||
if (!provider) {
|
||||
throw new NoCopilotProviderAvailable();
|
||||
@@ -565,8 +569,8 @@ export class CopilotController implements BeforeApplicationShutdown {
|
||||
@Res() res: Response,
|
||||
@Query() params: Record<string, string>
|
||||
) {
|
||||
const { unsplashKey } = this.config.plugins.copilot || {};
|
||||
if (!unsplashKey) {
|
||||
const { key } = this.config.copilot.unsplash;
|
||||
if (!key) {
|
||||
throw new UnsplashIsNotConfigured();
|
||||
}
|
||||
|
||||
@@ -574,7 +578,7 @@ export class CopilotController implements BeforeApplicationShutdown {
|
||||
const response = await fetch(
|
||||
`https://api.unsplash.com/search/photos?${query}`,
|
||||
{
|
||||
headers: { Authorization: `Client-ID ${unsplashKey}` },
|
||||
headers: { Authorization: `Client-ID ${key}` },
|
||||
signal: this.getSignal(req),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import './config';
|
||||
|
||||
import { ServerFeature } from '../../core/config';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ServerConfigModule } from '../../core';
|
||||
import { DocStorageModule } from '../../core/doc';
|
||||
import { FeatureModule } from '../../core/features';
|
||||
import { PermissionModule } from '../../core/permission';
|
||||
import { QuotaModule } from '../../core/quota';
|
||||
import { Plugin } from '../registry';
|
||||
import {
|
||||
CopilotContextDocJob,
|
||||
CopilotContextResolver,
|
||||
@@ -15,15 +16,7 @@ import {
|
||||
import { CopilotController } from './controller';
|
||||
import { ChatMessageCache } from './message';
|
||||
import { PromptService } from './prompt';
|
||||
import {
|
||||
assertProvidersConfigs,
|
||||
CopilotProviderService,
|
||||
FalProvider,
|
||||
OpenAIProvider,
|
||||
PerplexityProvider,
|
||||
registerCopilotProvider,
|
||||
} from './providers';
|
||||
import { GoogleProvider } from './providers/google';
|
||||
import { CopilotProviderFactory, CopilotProviders } from './providers';
|
||||
import {
|
||||
CopilotResolver,
|
||||
PromptsManagementResolver,
|
||||
@@ -37,42 +30,39 @@ import {
|
||||
} from './transcript';
|
||||
import { CopilotWorkflowExecutors, CopilotWorkflowService } from './workflow';
|
||||
|
||||
registerCopilotProvider(FalProvider);
|
||||
registerCopilotProvider(OpenAIProvider);
|
||||
registerCopilotProvider(GoogleProvider);
|
||||
registerCopilotProvider(PerplexityProvider);
|
||||
|
||||
@Plugin({
|
||||
name: 'copilot',
|
||||
imports: [DocStorageModule, FeatureModule, QuotaModule, PermissionModule],
|
||||
@Module({
|
||||
imports: [
|
||||
DocStorageModule,
|
||||
FeatureModule,
|
||||
QuotaModule,
|
||||
PermissionModule,
|
||||
ServerConfigModule,
|
||||
],
|
||||
providers: [
|
||||
// providers
|
||||
...CopilotProviders,
|
||||
CopilotProviderFactory,
|
||||
// services
|
||||
ChatSessionService,
|
||||
CopilotResolver,
|
||||
ChatMessageCache,
|
||||
UserCopilotResolver,
|
||||
PromptService,
|
||||
CopilotProviderService,
|
||||
CopilotStorage,
|
||||
PromptsManagementResolver,
|
||||
// workflow
|
||||
CopilotWorkflowService,
|
||||
...CopilotWorkflowExecutors,
|
||||
// context
|
||||
CopilotContextRootResolver,
|
||||
CopilotContextResolver,
|
||||
CopilotContextService,
|
||||
CopilotContextDocJob,
|
||||
// transcription
|
||||
CopilotTranscriptionService,
|
||||
CopilotTranscriptionResolver,
|
||||
// gql resolvers
|
||||
UserCopilotResolver,
|
||||
PromptsManagementResolver,
|
||||
CopilotContextRootResolver,
|
||||
],
|
||||
controllers: [CopilotController],
|
||||
contributesTo: ServerFeature.Copilot,
|
||||
if: config => {
|
||||
if (config.flavor.graphql || config.flavor.doc) {
|
||||
return assertProvidersConfigs(config);
|
||||
}
|
||||
return false;
|
||||
},
|
||||
})
|
||||
export class CopilotModule {}
|
||||
|
||||
@@ -3,12 +3,8 @@ import { Logger } from '@nestjs/common';
|
||||
import { AiPrompt } from '@prisma/client';
|
||||
import Mustache from 'mustache';
|
||||
|
||||
import {
|
||||
getTokenEncoder,
|
||||
PromptConfig,
|
||||
PromptMessage,
|
||||
PromptParams,
|
||||
} from '../types';
|
||||
import { PromptConfig, PromptMessage, PromptParams } from '../providers';
|
||||
import { getTokenEncoder } from '../types';
|
||||
|
||||
// disable escaping
|
||||
Mustache.escape = (text: string) => text;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { AiPrompt, PrismaClient } from '@prisma/client';
|
||||
|
||||
import { PromptConfig, PromptMessage } from '../types';
|
||||
import { PromptConfig, PromptMessage } from '../providers';
|
||||
|
||||
type Prompt = Omit<
|
||||
AiPrompt,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
import {
|
||||
@@ -6,17 +6,17 @@ import {
|
||||
PromptConfigSchema,
|
||||
PromptMessage,
|
||||
PromptMessageSchema,
|
||||
} from '../types';
|
||||
} from '../providers';
|
||||
import { ChatPrompt } from './chat-prompt';
|
||||
import { refreshPrompts } from './prompts';
|
||||
|
||||
@Injectable()
|
||||
export class PromptService implements OnModuleInit {
|
||||
export class PromptService implements OnApplicationBootstrap {
|
||||
private readonly cache = new Map<string, ChatPrompt>();
|
||||
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async onModuleInit() {
|
||||
async onApplicationBootstrap() {
|
||||
this.cache.clear();
|
||||
await refreshPrompts(this.db);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { ServerFeature, ServerService } from '../../../core';
|
||||
import type { FalProvider } from './fal';
|
||||
import type { GeminiProvider } from './gemini';
|
||||
import type { OpenAIProvider } from './openai';
|
||||
import type { PerplexityProvider } from './perplexity';
|
||||
import type { CopilotProvider } from './provider';
|
||||
import {
|
||||
CapabilityToCopilotProvider,
|
||||
CopilotCapability,
|
||||
CopilotProviderType,
|
||||
} from './types';
|
||||
|
||||
type TypedProvider = {
|
||||
[CopilotProviderType.Gemini]: GeminiProvider;
|
||||
[CopilotProviderType.OpenAI]: OpenAIProvider;
|
||||
[CopilotProviderType.Perplexity]: PerplexityProvider;
|
||||
[CopilotProviderType.FAL]: FalProvider;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class CopilotProviderFactory {
|
||||
constructor(private readonly server: ServerService) {}
|
||||
|
||||
private readonly logger = new Logger(CopilotProviderFactory.name);
|
||||
|
||||
readonly #providers = new Map<CopilotProviderType, CopilotProvider>();
|
||||
|
||||
getProvider<P extends CopilotProviderType>(provider: P): TypedProvider[P] {
|
||||
return this.#providers.get(provider) as TypedProvider[P];
|
||||
}
|
||||
|
||||
async getProviderByCapability<C extends CopilotCapability>(
|
||||
capability: C,
|
||||
filter: {
|
||||
model?: string;
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<CapabilityToCopilotProvider[C] | null> {
|
||||
this.logger.debug(
|
||||
`Resolving copilot provider for capability: ${capability}`
|
||||
);
|
||||
let candidate: CopilotProvider | null = null;
|
||||
for (const [type, provider] of this.#providers.entries()) {
|
||||
// we firstly match by capability
|
||||
if (provider.capabilities.includes(capability)) {
|
||||
// use the first match if no filter provided
|
||||
if (!filter.model && !filter.prefer) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
(!filter.model || (await provider.isModelAvailable(filter.model))) &&
|
||||
(!filter.prefer || filter.prefer === type)
|
||||
) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidate as CapabilityToCopilotProvider[C] | null;
|
||||
}
|
||||
|
||||
async getProviderByModel<C extends CopilotCapability>(
|
||||
model: string,
|
||||
filter: {
|
||||
prefer?: CopilotProviderType;
|
||||
} = {}
|
||||
): Promise<CapabilityToCopilotProvider[C] | null> {
|
||||
this.logger.debug(`Resolving copilot provider for model: ${model}`);
|
||||
|
||||
let candidate: CopilotProvider | null = null;
|
||||
for (const [type, provider] of this.#providers.entries()) {
|
||||
// we firstly match by model
|
||||
if (await provider.isModelAvailable(model)) {
|
||||
candidate = provider;
|
||||
this.logger.debug(`Copilot provider candidate found: ${type}`);
|
||||
|
||||
// then we match by prefer filter
|
||||
if (!filter.prefer || filter.prefer === type) {
|
||||
candidate = provider;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidate as CapabilityToCopilotProvider[C] | null;
|
||||
}
|
||||
|
||||
register(provider: CopilotProvider) {
|
||||
this.#providers.set(provider.type, provider);
|
||||
this.logger.log(`Copilot provider [${provider.type}] registered.`);
|
||||
this.server.enableFeature(ServerFeature.Copilot);
|
||||
}
|
||||
|
||||
unregister(provider: CopilotProvider) {
|
||||
this.#providers.delete(provider.type);
|
||||
this.logger.log(`Copilot provider [${provider.type}] unregistered.`);
|
||||
if (this.#providers.size === 0) {
|
||||
this.server.disableFeature(ServerFeature.Copilot);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,8 @@
|
||||
import assert from 'node:assert';
|
||||
|
||||
import {
|
||||
config as falConfig,
|
||||
stream as falStream,
|
||||
} from '@fal-ai/serverless-client';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { z, ZodType } from 'zod';
|
||||
|
||||
import {
|
||||
@@ -12,6 +11,7 @@ import {
|
||||
metrics,
|
||||
UserFriendlyError,
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
CopilotCapability,
|
||||
CopilotChatOptions,
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
CopilotProviderType,
|
||||
CopilotTextToImageProvider,
|
||||
PromptMessage,
|
||||
} from '../types';
|
||||
} from './types';
|
||||
|
||||
export type FalConfig = {
|
||||
apiKey: string;
|
||||
@@ -71,17 +71,19 @@ type FalPrompt = {
|
||||
}[];
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class FalProvider
|
||||
extends CopilotProvider<FalConfig>
|
||||
implements CopilotTextToImageProvider, CopilotImageToImageProvider
|
||||
{
|
||||
static readonly type = CopilotProviderType.FAL;
|
||||
static readonly capabilities = [
|
||||
override type = CopilotProviderType.FAL;
|
||||
override readonly capabilities = [
|
||||
CopilotCapability.TextToImage,
|
||||
CopilotCapability.ImageToImage,
|
||||
CopilotCapability.ImageToText,
|
||||
];
|
||||
|
||||
readonly availableModels = [
|
||||
override readonly models = [
|
||||
// text to image
|
||||
'fast-turbo-diffusion',
|
||||
// image to image
|
||||
@@ -96,27 +98,15 @@ export class FalProvider
|
||||
'llava-next',
|
||||
];
|
||||
|
||||
constructor(private readonly config: FalConfig) {
|
||||
assert(FalProvider.assetsConfig(config));
|
||||
override configured(): boolean {
|
||||
return !!this.config.apiKey;
|
||||
}
|
||||
|
||||
protected override setup() {
|
||||
super.setup();
|
||||
falConfig({ credentials: this.config.apiKey });
|
||||
}
|
||||
|
||||
static assetsConfig(config: FalConfig) {
|
||||
return !!config.apiKey;
|
||||
}
|
||||
|
||||
get type(): CopilotProviderType {
|
||||
return FalProvider.type;
|
||||
}
|
||||
|
||||
getCapabilities(): CopilotCapability[] {
|
||||
return FalProvider.capabilities;
|
||||
}
|
||||
|
||||
async isModelAvailable(model: string): Promise<boolean> {
|
||||
return this.availableModels.includes(model);
|
||||
}
|
||||
|
||||
private extractArray<T>(value: T | T[] | undefined): T[] {
|
||||
if (Array.isArray(value)) return value;
|
||||
return value ? [value] : [];
|
||||
@@ -211,7 +201,7 @@ export class FalProvider
|
||||
model: string = 'llava-next',
|
||||
options: CopilotChatOptions = {}
|
||||
): Promise<string> {
|
||||
if (!this.availableModels.includes(model)) {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
}
|
||||
|
||||
@@ -269,7 +259,7 @@ export class FalProvider
|
||||
|
||||
private async buildResponse(
|
||||
messages: PromptMessage[],
|
||||
model: string = this.availableModels[0],
|
||||
model: string = this.models[0],
|
||||
options: CopilotImageOptions = {}
|
||||
) {
|
||||
// by default, image prompt assumes there is only one message
|
||||
@@ -300,10 +290,10 @@ export class FalProvider
|
||||
// ====== image to image ======
|
||||
async generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string = this.availableModels[0],
|
||||
model: string = this.models[0],
|
||||
options: CopilotImageOptions = {}
|
||||
): Promise<Array<string>> {
|
||||
if (!this.availableModels.includes(model)) {
|
||||
if (!(await this.isModelAvailable(model))) {
|
||||
throw new CopilotPromptInvalid(`Invalid model: ${model}`);
|
||||
}
|
||||
|
||||
@@ -333,7 +323,7 @@ export class FalProvider
|
||||
|
||||
async *generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model: string = this.availableModels[0],
|
||||
model: string = this.models[0],
|
||||
options: CopilotImageOptions = {}
|
||||
): AsyncIterable<string> {
|
||||
try {
|
||||
|
||||
+21
-28
@@ -2,7 +2,6 @@ import {
|
||||
createGoogleGenerativeAI,
|
||||
type GoogleGenerativeAIProvider,
|
||||
} from '@ai-sdk/google';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import {
|
||||
AISDKError,
|
||||
type CoreAssistantMessage,
|
||||
@@ -19,6 +18,7 @@ import {
|
||||
metrics,
|
||||
UserFriendlyError,
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
ChatMessageRole,
|
||||
CopilotCapability,
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
CopilotProviderType,
|
||||
CopilotTextToTextProvider,
|
||||
PromptMessage,
|
||||
} from '../types';
|
||||
} from './types';
|
||||
|
||||
export const DEFAULT_DIMENSIONS = 256;
|
||||
|
||||
@@ -49,45 +49,38 @@ const FORMAT_INFER_MAP: Record<string, string> = {
|
||||
flv: 'video/flv',
|
||||
};
|
||||
|
||||
export type GoogleConfig = {
|
||||
export type GeminiConfig = {
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
};
|
||||
|
||||
type ChatMessage = CoreUserMessage | CoreAssistantMessage;
|
||||
|
||||
export class GoogleProvider implements CopilotTextToTextProvider {
|
||||
static readonly type = CopilotProviderType.Google;
|
||||
static readonly capabilities = [CopilotCapability.TextToText];
|
||||
|
||||
readonly availableModels = [
|
||||
export class GeminiProvider
|
||||
extends CopilotProvider<GeminiConfig>
|
||||
implements CopilotTextToTextProvider
|
||||
{
|
||||
override readonly type = CopilotProviderType.Gemini;
|
||||
override readonly capabilities = [CopilotCapability.TextToText];
|
||||
override readonly models = [
|
||||
// text to text
|
||||
'gemini-2.0-flash-001',
|
||||
// embeddings
|
||||
'text-embedding-004',
|
||||
];
|
||||
|
||||
private readonly logger = new Logger(GoogleProvider.name);
|
||||
private readonly instance: GoogleGenerativeAIProvider;
|
||||
#instance!: GoogleGenerativeAIProvider;
|
||||
|
||||
constructor(config: GoogleConfig) {
|
||||
this.instance = createGoogleGenerativeAI(config);
|
||||
override configured(): boolean {
|
||||
return !!this.config.apiKey;
|
||||
}
|
||||
|
||||
static assetsConfig(config: GoogleConfig) {
|
||||
return !!config?.apiKey;
|
||||
}
|
||||
|
||||
get type(): CopilotProviderType {
|
||||
return GoogleProvider.type;
|
||||
}
|
||||
|
||||
getCapabilities(): CopilotCapability[] {
|
||||
return GoogleProvider.capabilities;
|
||||
}
|
||||
|
||||
async isModelAvailable(model: string): Promise<boolean> {
|
||||
return this.availableModels.includes(model);
|
||||
protected override setup() {
|
||||
super.setup();
|
||||
this.#instance = createGoogleGenerativeAI({
|
||||
apiKey: this.config.apiKey,
|
||||
baseURL: this.config.baseUrl,
|
||||
});
|
||||
}
|
||||
|
||||
private inferMimeType(url: string) {
|
||||
@@ -239,7 +232,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
|
||||
const [system, msgs] = await this.chatToGPTMessage(messages);
|
||||
|
||||
const { text } = await generateText({
|
||||
model: this.instance(model, {
|
||||
model: this.#instance(model, {
|
||||
audioTimestamp: Boolean(options.audioTimestamp),
|
||||
structuredOutputs: Boolean(options.jsonMode),
|
||||
}),
|
||||
@@ -268,7 +261,7 @@ export class GoogleProvider implements CopilotTextToTextProvider {
|
||||
const [system, msgs] = await this.chatToGPTMessage(messages);
|
||||
|
||||
const { textStream } = streamText({
|
||||
model: this.instance(model),
|
||||
model: this.#instance(model),
|
||||
system,
|
||||
messages: msgs,
|
||||
abortSignal: options.signal,
|
||||
@@ -1,203 +1,18 @@
|
||||
import assert from 'node:assert';
|
||||
import { FalProvider } from './fal';
|
||||
import { GeminiProvider } from './gemini';
|
||||
import { OpenAIProvider } from './openai';
|
||||
import { PerplexityProvider } from './perplexity';
|
||||
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { AFFiNEConfig, Config } from '../../../base';
|
||||
import { CopilotStartupConfigurations } from '../config';
|
||||
import {
|
||||
CapabilityToCopilotProvider,
|
||||
CopilotCapability,
|
||||
CopilotProvider,
|
||||
CopilotProviderType,
|
||||
} from '../types';
|
||||
|
||||
type CopilotProviderConfig =
|
||||
CopilotStartupConfigurations[keyof CopilotStartupConfigurations];
|
||||
|
||||
interface CopilotProviderDefinition<C extends CopilotProviderConfig> {
|
||||
// constructor signature
|
||||
new (config: C): CopilotProvider;
|
||||
// type of the provider
|
||||
readonly type: CopilotProviderType;
|
||||
// capabilities of the provider, like text to text, text to image, etc.
|
||||
readonly capabilities: CopilotCapability[];
|
||||
// asserts that the config is valid for this provider
|
||||
assetsConfig(config: C): boolean;
|
||||
}
|
||||
|
||||
// registered provider factory
|
||||
const COPILOT_PROVIDER = new Map<
|
||||
CopilotProviderType,
|
||||
(config: Config, logger: Logger) => CopilotProvider
|
||||
>();
|
||||
|
||||
// map of capabilities to providers
|
||||
const PROVIDER_CAPABILITY_MAP = new Map<
|
||||
CopilotCapability,
|
||||
CopilotProviderType[]
|
||||
>();
|
||||
|
||||
// config assertions for providers
|
||||
const ASSERT_CONFIG = new Map<
|
||||
CopilotProviderType,
|
||||
(config: AFFiNEConfig) => void
|
||||
>();
|
||||
|
||||
export function registerCopilotProvider<
|
||||
C extends CopilotProviderConfig = CopilotProviderConfig,
|
||||
>(provider: CopilotProviderDefinition<C>) {
|
||||
const type = provider.type;
|
||||
|
||||
const factory = (config: Config, logger: Logger) => {
|
||||
const providerConfig = config.plugins.copilot?.[type];
|
||||
if (!provider.assetsConfig(providerConfig as C)) {
|
||||
throw new Error(
|
||||
`Invalid configuration for copilot provider ${type}: ${JSON.stringify(providerConfig)}`
|
||||
);
|
||||
}
|
||||
const instance = new provider(providerConfig as C);
|
||||
logger.debug(
|
||||
`Copilot provider ${type} registered, capabilities: ${provider.capabilities.join(', ')}`
|
||||
);
|
||||
|
||||
return instance;
|
||||
};
|
||||
// register the provider
|
||||
COPILOT_PROVIDER.set(type, factory);
|
||||
// register the provider capabilities
|
||||
for (const capability of provider.capabilities) {
|
||||
const providers = PROVIDER_CAPABILITY_MAP.get(capability) || [];
|
||||
if (!providers.includes(type)) {
|
||||
providers.push(type);
|
||||
}
|
||||
PROVIDER_CAPABILITY_MAP.set(capability, providers);
|
||||
}
|
||||
// register the provider config assertion
|
||||
ASSERT_CONFIG.set(type, (config: AFFiNEConfig) => {
|
||||
assert(config.plugins.copilot);
|
||||
const providerConfig = config.plugins.copilot[type];
|
||||
if (!providerConfig) return false;
|
||||
return provider.assetsConfig(providerConfig as C);
|
||||
});
|
||||
}
|
||||
|
||||
export function unregisterCopilotProvider(type: CopilotProviderType) {
|
||||
COPILOT_PROVIDER.delete(type);
|
||||
ASSERT_CONFIG.delete(type);
|
||||
for (const providers of PROVIDER_CAPABILITY_MAP.values()) {
|
||||
const index = providers.indexOf(type);
|
||||
if (index !== -1) {
|
||||
providers.splice(index, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Asserts that the config is valid for any registered providers
|
||||
export function assertProvidersConfigs(config: AFFiNEConfig) {
|
||||
return Array.from(ASSERT_CONFIG.values()).some(assertConfig =>
|
||||
assertConfig(config)
|
||||
);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class CopilotProviderService {
|
||||
private readonly logger = new Logger(CopilotProviderService.name);
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
private readonly cachedProviders = new Map<
|
||||
CopilotProviderType,
|
||||
CopilotProvider
|
||||
>();
|
||||
|
||||
private create(provider: CopilotProviderType): CopilotProvider {
|
||||
assert(this.config.plugins.copilot);
|
||||
const providerFactory = COPILOT_PROVIDER.get(provider);
|
||||
|
||||
if (!providerFactory) {
|
||||
throw new Error(`Unknown copilot provider type: ${provider}`);
|
||||
}
|
||||
|
||||
return providerFactory(this.config, this.logger);
|
||||
}
|
||||
|
||||
getProvider(provider: CopilotProviderType): CopilotProvider | null {
|
||||
try {
|
||||
if (!this.cachedProviders.has(provider)) {
|
||||
this.cachedProviders.set(provider, this.create(provider));
|
||||
}
|
||||
return this.cachedProviders.get(provider) as CopilotProvider;
|
||||
} catch {
|
||||
// skip if the provider is not available
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async getProviderByCapability<C extends CopilotCapability>(
|
||||
capability: C,
|
||||
model?: string,
|
||||
prefer?: CopilotProviderType
|
||||
): Promise<CapabilityToCopilotProvider[C] | null> {
|
||||
const providers = PROVIDER_CAPABILITY_MAP.get(capability);
|
||||
if (Array.isArray(providers) && providers.length) {
|
||||
let selectedProvider: CopilotProviderType | undefined = prefer;
|
||||
let currentIndex = -1;
|
||||
|
||||
if (!selectedProvider) {
|
||||
currentIndex = 0;
|
||||
selectedProvider = providers[currentIndex];
|
||||
}
|
||||
|
||||
while (selectedProvider) {
|
||||
// find first provider that supports the capability and model
|
||||
if (providers.includes(selectedProvider)) {
|
||||
const provider = this.getProvider(selectedProvider);
|
||||
|
||||
if (provider && provider.getCapabilities().includes(capability)) {
|
||||
if (model) {
|
||||
if (await provider.isModelAvailable(model)) {
|
||||
return provider as CapabilityToCopilotProvider[C];
|
||||
}
|
||||
} else {
|
||||
return provider as CapabilityToCopilotProvider[C];
|
||||
}
|
||||
}
|
||||
}
|
||||
currentIndex += 1;
|
||||
selectedProvider = providers[currentIndex];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async getProviderByModel<C extends CopilotCapability>(
|
||||
model: string,
|
||||
prefer?: CopilotProviderType
|
||||
): Promise<CapabilityToCopilotProvider[C] | null> {
|
||||
const providers = Array.from(COPILOT_PROVIDER.keys());
|
||||
if (providers.length) {
|
||||
let selectedProvider: CopilotProviderType | undefined = prefer;
|
||||
let currentIndex = -1;
|
||||
|
||||
if (!selectedProvider) {
|
||||
currentIndex = 0;
|
||||
selectedProvider = providers[currentIndex];
|
||||
}
|
||||
|
||||
while (selectedProvider) {
|
||||
const provider = this.getProvider(selectedProvider);
|
||||
|
||||
if (provider && (await provider.isModelAvailable(model))) {
|
||||
return provider as CapabilityToCopilotProvider[C];
|
||||
}
|
||||
|
||||
currentIndex += 1;
|
||||
selectedProvider = providers[currentIndex];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
export const CopilotProviders = [
|
||||
OpenAIProvider,
|
||||
FalProvider,
|
||||
GeminiProvider,
|
||||
PerplexityProvider,
|
||||
];
|
||||
|
||||
export { CopilotProviderFactory } from './factory';
|
||||
export { FalProvider } from './fal';
|
||||
export { GeminiProvider } from './gemini';
|
||||
export { OpenAIProvider } from './openai';
|
||||
export { PerplexityProvider } from './perplexity';
|
||||
export * from './types';
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { APIError, BadRequestError, ClientOptions, OpenAI } from 'openai';
|
||||
|
||||
import {
|
||||
@@ -7,6 +6,7 @@ import {
|
||||
metrics,
|
||||
UserFriendlyError,
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
ChatMessageRole,
|
||||
CopilotCapability,
|
||||
@@ -19,28 +19,31 @@ import {
|
||||
CopilotTextToImageProvider,
|
||||
CopilotTextToTextProvider,
|
||||
PromptMessage,
|
||||
} from '../types';
|
||||
} from './types';
|
||||
|
||||
export const DEFAULT_DIMENSIONS = 256;
|
||||
|
||||
const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
|
||||
|
||||
export type OpenAIConfig = ClientOptions;
|
||||
|
||||
export class OpenAIProvider
|
||||
extends CopilotProvider<OpenAIConfig>
|
||||
implements
|
||||
CopilotTextToTextProvider,
|
||||
CopilotTextToEmbeddingProvider,
|
||||
CopilotTextToImageProvider,
|
||||
CopilotImageToTextProvider
|
||||
{
|
||||
static readonly type = CopilotProviderType.OpenAI;
|
||||
static readonly capabilities = [
|
||||
readonly type = CopilotProviderType.OpenAI;
|
||||
readonly capabilities = [
|
||||
CopilotCapability.TextToText,
|
||||
CopilotCapability.TextToEmbedding,
|
||||
CopilotCapability.TextToImage,
|
||||
CopilotCapability.ImageToText,
|
||||
];
|
||||
|
||||
readonly availableModels = [
|
||||
readonly models = [
|
||||
// text to text
|
||||
'gpt-4o',
|
||||
'gpt-4o-2024-08-06',
|
||||
@@ -59,41 +62,32 @@ export class OpenAIProvider
|
||||
'dall-e-3',
|
||||
];
|
||||
|
||||
private readonly logger = new Logger(OpenAIProvider.type);
|
||||
private readonly instance: OpenAI;
|
||||
#existsModels: string[] = [];
|
||||
#instance!: OpenAI;
|
||||
|
||||
private existsModels: string[] | undefined;
|
||||
|
||||
constructor(config: ClientOptions) {
|
||||
this.instance = new OpenAI(config);
|
||||
override configured(): boolean {
|
||||
return !!this.config.apiKey;
|
||||
}
|
||||
|
||||
static assetsConfig(config: ClientOptions) {
|
||||
return !!config?.apiKey;
|
||||
protected override setup() {
|
||||
super.setup();
|
||||
this.#instance = new OpenAI(this.config);
|
||||
}
|
||||
|
||||
get type(): CopilotProviderType {
|
||||
return OpenAIProvider.type;
|
||||
}
|
||||
|
||||
getCapabilities(): CopilotCapability[] {
|
||||
return OpenAIProvider.capabilities;
|
||||
}
|
||||
|
||||
async isModelAvailable(model: string): Promise<boolean> {
|
||||
const knownModels = this.availableModels.includes(model);
|
||||
override async isModelAvailable(model: string): Promise<boolean> {
|
||||
const knownModels = this.models.includes(model);
|
||||
if (knownModels) return true;
|
||||
|
||||
if (!this.existsModels) {
|
||||
if (!this.#existsModels) {
|
||||
try {
|
||||
this.existsModels = await this.instance.models
|
||||
this.#existsModels = await this.#instance.models
|
||||
.list()
|
||||
.then(({ data }) => data.map(m => m.id));
|
||||
} catch (e: any) {
|
||||
this.logger.error('Failed to fetch online model list', e.stack);
|
||||
}
|
||||
}
|
||||
return !!this.existsModels?.includes(model);
|
||||
return !!this.#existsModels?.includes(model);
|
||||
}
|
||||
|
||||
protected chatToGPTMessage(
|
||||
@@ -226,7 +220,7 @@ export class OpenAIProvider
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_calls').add(1, { model });
|
||||
const result = await this.instance.chat.completions.create(
|
||||
const result = await this.#instance.chat.completions.create(
|
||||
{
|
||||
messages: this.chatToGPTMessage(messages),
|
||||
model: model,
|
||||
@@ -257,7 +251,7 @@ export class OpenAIProvider
|
||||
|
||||
try {
|
||||
metrics.ai.counter('chat_text_stream_calls').add(1, { model });
|
||||
const result = await this.instance.chat.completions.create(
|
||||
const result = await this.#instance.chat.completions.create(
|
||||
{
|
||||
stream: true,
|
||||
messages: this.chatToGPTMessage(messages),
|
||||
@@ -307,7 +301,7 @@ export class OpenAIProvider
|
||||
|
||||
try {
|
||||
metrics.ai.counter('generate_embedding_calls').add(1, { model });
|
||||
const result = await this.instance.embeddings.create({
|
||||
const result = await this.#instance.embeddings.create({
|
||||
model: model,
|
||||
input: messages,
|
||||
dimensions: options.dimensions || DEFAULT_DIMENSIONS,
|
||||
@@ -333,7 +327,7 @@ export class OpenAIProvider
|
||||
|
||||
try {
|
||||
metrics.ai.counter('generate_images_calls').add(1, { model });
|
||||
const result = await this.instance.images.generate(
|
||||
const result = await this.#instance.images.generate(
|
||||
{
|
||||
prompt,
|
||||
model,
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import assert from 'node:assert';
|
||||
|
||||
import { EventSourceParserStream } from 'eventsource-parser/stream';
|
||||
import { z } from 'zod';
|
||||
|
||||
@@ -8,13 +6,14 @@ import {
|
||||
CopilotProviderSideError,
|
||||
metrics,
|
||||
} from '../../../base';
|
||||
import { CopilotProvider } from './provider';
|
||||
import {
|
||||
CopilotCapability,
|
||||
CopilotChatOptions,
|
||||
CopilotProviderType,
|
||||
CopilotTextToTextProvider,
|
||||
PromptMessage,
|
||||
} from '../types';
|
||||
} from './types';
|
||||
|
||||
export type PerplexityConfig = {
|
||||
apiKey: string;
|
||||
@@ -164,36 +163,21 @@ export class CitationParser {
|
||||
}
|
||||
}
|
||||
|
||||
export class PerplexityProvider implements CopilotTextToTextProvider {
|
||||
static readonly type = CopilotProviderType.Perplexity;
|
||||
|
||||
static readonly capabilities = [CopilotCapability.TextToText];
|
||||
|
||||
static assetsConfig(config: PerplexityConfig) {
|
||||
return !!config.apiKey;
|
||||
}
|
||||
|
||||
constructor(private readonly config: PerplexityConfig) {
|
||||
assert(PerplexityProvider.assetsConfig(config));
|
||||
}
|
||||
|
||||
readonly availableModels = [
|
||||
export class PerplexityProvider
|
||||
extends CopilotProvider<PerplexityConfig>
|
||||
implements CopilotTextToTextProvider
|
||||
{
|
||||
readonly type = CopilotProviderType.Perplexity;
|
||||
readonly capabilities = [CopilotCapability.TextToText];
|
||||
readonly models = [
|
||||
'sonar',
|
||||
'sonar-pro',
|
||||
'sonar-reasoning',
|
||||
'sonar-reasoning-pro',
|
||||
];
|
||||
|
||||
get type(): CopilotProviderType {
|
||||
return PerplexityProvider.type;
|
||||
}
|
||||
|
||||
getCapabilities(): CopilotCapability[] {
|
||||
return PerplexityProvider.capabilities;
|
||||
}
|
||||
|
||||
async isModelAvailable(model: string): Promise<boolean> {
|
||||
return this.availableModels.includes(model);
|
||||
override configured(): boolean {
|
||||
return !!this.config.apiKey;
|
||||
}
|
||||
|
||||
async generateText(
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { Config, OnEvent } from '../../../base';
|
||||
import { CopilotProviderFactory } from './factory';
|
||||
import { CopilotCapability, CopilotProviderType } from './types';
|
||||
|
||||
@Injectable()
|
||||
export abstract class CopilotProvider<C = any> {
|
||||
protected readonly logger = new Logger(this.constructor.name);
|
||||
abstract readonly type: CopilotProviderType;
|
||||
abstract readonly capabilities: CopilotCapability[];
|
||||
abstract readonly models: string[];
|
||||
abstract configured(): boolean;
|
||||
|
||||
@Inject() protected readonly AFFiNEConfig!: Config;
|
||||
@Inject() protected readonly factory!: CopilotProviderFactory;
|
||||
|
||||
get config(): C {
|
||||
return this.AFFiNEConfig.copilot.providers[this.type] as C;
|
||||
}
|
||||
|
||||
isModelAvailable(model: string): Promise<boolean> | boolean {
|
||||
return this.models.includes(model);
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
this.setup();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged(event: Events['config.changed']) {
|
||||
if ('copilot' in event.updates) {
|
||||
this.setup();
|
||||
}
|
||||
}
|
||||
|
||||
protected setup() {
|
||||
if (this.configured()) {
|
||||
this.factory.register(this);
|
||||
} else {
|
||||
this.factory.unregister(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { type CopilotProvider } from './provider';
|
||||
|
||||
export enum CopilotProviderType {
|
||||
FAL = 'fal',
|
||||
Gemini = 'gemini',
|
||||
OpenAI = 'openai',
|
||||
Perplexity = 'perplexity',
|
||||
}
|
||||
|
||||
export enum CopilotCapability {
|
||||
TextToText = 'text-to-text',
|
||||
TextToEmbedding = 'text-to-embedding',
|
||||
TextToImage = 'text-to-image',
|
||||
ImageToImage = 'image-to-image',
|
||||
ImageToText = 'image-to-text',
|
||||
}
|
||||
|
||||
export const PromptConfigStrictSchema = z.object({
|
||||
// openai
|
||||
jsonMode: z.boolean().nullable().optional(),
|
||||
frequencyPenalty: z.number().nullable().optional(),
|
||||
presencePenalty: z.number().nullable().optional(),
|
||||
temperature: z.number().nullable().optional(),
|
||||
topP: z.number().nullable().optional(),
|
||||
maxTokens: z.number().nullable().optional(),
|
||||
// fal
|
||||
modelName: z.string().nullable().optional(),
|
||||
loras: z
|
||||
.array(
|
||||
z.object({ path: z.string(), scale: z.number().nullable().optional() })
|
||||
)
|
||||
.nullable()
|
||||
.optional(),
|
||||
// google
|
||||
audioTimestamp: z.boolean().nullable().optional(),
|
||||
});
|
||||
|
||||
export const PromptConfigSchema =
|
||||
PromptConfigStrictSchema.nullable().optional();
|
||||
|
||||
export type PromptConfig = z.infer<typeof PromptConfigSchema>;
|
||||
|
||||
export const ChatMessageRole = Object.values(AiPromptRole) as [
|
||||
'system',
|
||||
'assistant',
|
||||
'user',
|
||||
];
|
||||
|
||||
export const PureMessageSchema = z.object({
|
||||
content: z.string(),
|
||||
attachments: z.array(z.string()).optional().nullable(),
|
||||
params: z.record(z.any()).optional().nullable(),
|
||||
});
|
||||
|
||||
export const PromptMessageSchema = PureMessageSchema.extend({
|
||||
role: z.enum(ChatMessageRole),
|
||||
}).strict();
|
||||
export type PromptMessage = z.infer<typeof PromptMessageSchema>;
|
||||
export type PromptParams = NonNullable<PromptMessage['params']>;
|
||||
|
||||
const CopilotProviderOptionsSchema = z.object({
|
||||
signal: z.instanceof(AbortSignal).optional(),
|
||||
user: z.string().optional(),
|
||||
});
|
||||
|
||||
const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
PromptConfigStrictSchema
|
||||
).optional();
|
||||
|
||||
export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
|
||||
|
||||
const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({
|
||||
dimensions: z.number(),
|
||||
}).optional();
|
||||
|
||||
export type CopilotEmbeddingOptions = z.infer<
|
||||
typeof CopilotEmbeddingOptionsSchema
|
||||
>;
|
||||
|
||||
const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
PromptConfigStrictSchema
|
||||
)
|
||||
.extend({
|
||||
seed: z.number().optional(),
|
||||
})
|
||||
.optional();
|
||||
|
||||
export type CopilotImageOptions = z.infer<typeof CopilotImageOptionsSchema>;
|
||||
|
||||
export interface CopilotTextToTextProvider extends CopilotProvider {
|
||||
generateText(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
generateTextStream(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export interface CopilotTextToEmbeddingProvider extends CopilotProvider {
|
||||
generateEmbedding(
|
||||
messages: string[] | string,
|
||||
model: string,
|
||||
options?: CopilotEmbeddingOptions
|
||||
): Promise<number[][]>;
|
||||
}
|
||||
|
||||
export interface CopilotTextToImageProvider extends CopilotProvider {
|
||||
generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): Promise<Array<string>>;
|
||||
generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotImageOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export interface CopilotImageToTextProvider extends CopilotProvider {
|
||||
generateText(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
generateTextStream(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export interface CopilotImageToImageProvider extends CopilotProvider {
|
||||
generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): Promise<Array<string>>;
|
||||
generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotImageOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export type CapabilityToCopilotProvider = {
|
||||
[CopilotCapability.TextToText]: CopilotTextToTextProvider;
|
||||
[CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
|
||||
[CopilotCapability.TextToImage]: CopilotTextToImageProvider;
|
||||
[CopilotCapability.ImageToText]: CopilotImageToTextProvider;
|
||||
[CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
|
||||
};
|
||||
|
||||
export type CopilotTextProvider =
|
||||
| CopilotTextToTextProvider
|
||||
| CopilotImageToTextProvider;
|
||||
export type CopilotImageProvider =
|
||||
| CopilotTextToImageProvider
|
||||
| CopilotImageToImageProvider;
|
||||
export type CopilotAllProvider =
|
||||
| CopilotTextProvider
|
||||
| CopilotImageProvider
|
||||
| CopilotTextToEmbeddingProvider;
|
||||
@@ -17,6 +17,7 @@ import { QuotaService } from '../../core/quota';
|
||||
import { Models } from '../../models';
|
||||
import { ChatMessageCache } from './message';
|
||||
import { PromptService } from './prompt';
|
||||
import { PromptMessage, PromptParams } from './providers';
|
||||
import {
|
||||
AvailableModel,
|
||||
ChatHistory,
|
||||
@@ -28,8 +29,6 @@ import {
|
||||
ChatSessionState,
|
||||
getTokenEncoder,
|
||||
ListHistoriesOptions,
|
||||
PromptMessage,
|
||||
PromptParams,
|
||||
SubmittedMessage,
|
||||
} from './types';
|
||||
|
||||
@@ -533,15 +532,17 @@ export class ChatSessionService {
|
||||
const ret = ChatMessageSchema.array().safeParse(messages);
|
||||
if (ret.success) {
|
||||
// render system prompt
|
||||
const preload = options?.withPrompt
|
||||
? prompt
|
||||
.finish(ret.data[0]?.params || {}, id)
|
||||
.filter(({ role }) => role !== 'system')
|
||||
: [];
|
||||
const preload = (
|
||||
options?.withPrompt
|
||||
? prompt
|
||||
.finish(ret.data[0]?.params || {}, id)
|
||||
.filter(({ role }) => role !== 'system')
|
||||
: []
|
||||
) as ChatMessage[];
|
||||
|
||||
// `createdAt` is required for history sorting in frontend
|
||||
// let's fake the creating time of prompt messages
|
||||
(preload as ChatMessage[]).forEach((msg, i) => {
|
||||
preload.forEach((msg, i) => {
|
||||
msg.createdAt = new Date(
|
||||
createdAt.getTime() - preload.length - i - 1
|
||||
);
|
||||
|
||||
@@ -25,9 +25,7 @@ export class CopilotStorage {
|
||||
private readonly storageFactory: StorageProviderFactory,
|
||||
private readonly quota: QuotaService
|
||||
) {
|
||||
this.provider = this.storageFactory.create(
|
||||
this.config.plugins.copilot.storage
|
||||
);
|
||||
this.provider = this.storageFactory.create(this.config.copilot.storage);
|
||||
}
|
||||
|
||||
@CallMetric('ai', 'blob_put')
|
||||
@@ -39,7 +37,7 @@ export class CopilotStorage {
|
||||
) {
|
||||
const name = `${userId}/${workspaceId}/${key}`;
|
||||
await this.provider.put(name, blob);
|
||||
if (this.config.node.dev || this.config.node.test) {
|
||||
if (!env.prod) {
|
||||
// return image base64url for dev environment
|
||||
return `data:image/png;base64,${blob.toString('base64')}`;
|
||||
}
|
||||
|
||||
@@ -11,13 +11,13 @@ import {
|
||||
} from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import { PromptService } from '../prompt';
|
||||
import { CopilotProviderService } from '../providers';
|
||||
import { CopilotStorage } from '../storage';
|
||||
import {
|
||||
CopilotCapability,
|
||||
CopilotProviderFactory,
|
||||
CopilotTextProvider,
|
||||
PromptMessage,
|
||||
} from '../types';
|
||||
} from '../providers';
|
||||
import { CopilotStorage } from '../storage';
|
||||
import {
|
||||
TranscriptionPayload,
|
||||
TranscriptionSchema,
|
||||
@@ -38,7 +38,7 @@ export class CopilotTranscriptionService {
|
||||
private readonly job: JobQueue,
|
||||
private readonly storage: CopilotStorage,
|
||||
private readonly prompt: PromptService,
|
||||
private readonly provider: CopilotProviderService
|
||||
private readonly providerFactory: CopilotProviderFactory
|
||||
) {}
|
||||
|
||||
async submitTranscriptionJob(
|
||||
@@ -123,9 +123,9 @@ export class CopilotTranscriptionService {
|
||||
}
|
||||
|
||||
private async getProvider(model: string): Promise<CopilotTextProvider> {
|
||||
let provider = await this.provider.getProviderByCapability(
|
||||
let provider = await this.providerFactory.getProviderByCapability(
|
||||
CopilotCapability.TextToText,
|
||||
model
|
||||
{ model }
|
||||
);
|
||||
|
||||
if (!provider) {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { type Tokenizer } from '@affine/server-native';
|
||||
import { AiPromptRole } from '@prisma/client';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { fromModelName } from '../../native';
|
||||
import type { ChatPrompt } from './prompt';
|
||||
import { PromptMessageSchema, PureMessageSchema } from './providers';
|
||||
|
||||
export enum AvailableModels {
|
||||
// text to text
|
||||
@@ -41,77 +41,30 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null {
|
||||
|
||||
// ======== ChatMessage ========
|
||||
|
||||
export const ChatMessageRole = Object.values(AiPromptRole) as [
|
||||
'system',
|
||||
'assistant',
|
||||
'user',
|
||||
];
|
||||
|
||||
const PureMessageSchema = z.object({
|
||||
content: z.string(),
|
||||
attachments: z.array(z.string()).optional().nullable(),
|
||||
params: z.record(z.any()).optional().nullable(),
|
||||
});
|
||||
|
||||
export const PromptMessageSchema = PureMessageSchema.extend({
|
||||
role: z.enum(ChatMessageRole),
|
||||
}).strict();
|
||||
|
||||
export type PromptMessage = z.infer<typeof PromptMessageSchema>;
|
||||
|
||||
export type PromptParams = NonNullable<PromptMessage['params']>;
|
||||
|
||||
export const PromptConfigStrictSchema = z.object({
|
||||
// openai
|
||||
jsonMode: z.boolean().nullable().optional(),
|
||||
frequencyPenalty: z.number().nullable().optional(),
|
||||
presencePenalty: z.number().nullable().optional(),
|
||||
temperature: z.number().nullable().optional(),
|
||||
topP: z.number().nullable().optional(),
|
||||
maxTokens: z.number().nullable().optional(),
|
||||
// fal
|
||||
modelName: z.string().nullable().optional(),
|
||||
loras: z
|
||||
.array(
|
||||
z.object({ path: z.string(), scale: z.number().nullable().optional() })
|
||||
)
|
||||
.nullable()
|
||||
.optional(),
|
||||
// google
|
||||
audioTimestamp: z.boolean().nullable().optional(),
|
||||
});
|
||||
|
||||
export const PromptConfigSchema =
|
||||
PromptConfigStrictSchema.nullable().optional();
|
||||
|
||||
export type PromptConfig = z.infer<typeof PromptConfigSchema>;
|
||||
|
||||
export const ChatMessageSchema = PromptMessageSchema.extend({
|
||||
id: z.string().optional(),
|
||||
createdAt: z.date(),
|
||||
}).strict();
|
||||
|
||||
export type ChatMessage = z.infer<typeof ChatMessageSchema>;
|
||||
|
||||
export const SubmittedMessageSchema = PureMessageSchema.extend({
|
||||
sessionId: z.string(),
|
||||
content: z.string().optional(),
|
||||
}).strict();
|
||||
|
||||
export type SubmittedMessage = z.infer<typeof SubmittedMessageSchema>;
|
||||
|
||||
export const ChatHistorySchema = z
|
||||
.object({
|
||||
sessionId: z.string(),
|
||||
action: z.string().nullable(),
|
||||
tokens: z.number(),
|
||||
messages: z.array(PromptMessageSchema.or(ChatMessageSchema)),
|
||||
messages: z.array(ChatMessageSchema),
|
||||
createdAt: z.date(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type ChatHistory = z.infer<typeof ChatHistorySchema>;
|
||||
|
||||
export const SubmittedMessageSchema = PureMessageSchema.extend({
|
||||
sessionId: z.string(),
|
||||
content: z.string().optional(),
|
||||
}).strict();
|
||||
export type SubmittedMessage = z.infer<typeof SubmittedMessageSchema>;
|
||||
|
||||
// ======== Chat Session ========
|
||||
|
||||
export interface ChatSessionOptions {
|
||||
@@ -154,142 +107,9 @@ export type ListHistoriesOptions = {
|
||||
withPrompt: boolean | undefined;
|
||||
};
|
||||
|
||||
// ======== Provider Interface ========
|
||||
|
||||
export enum CopilotProviderType {
|
||||
FAL = 'fal',
|
||||
Google = 'google',
|
||||
OpenAI = 'openai',
|
||||
Perplexity = 'perplexity',
|
||||
// only for test
|
||||
Test = 'test',
|
||||
}
|
||||
|
||||
export enum CopilotCapability {
|
||||
TextToText = 'text-to-text',
|
||||
TextToEmbedding = 'text-to-embedding',
|
||||
TextToImage = 'text-to-image',
|
||||
ImageToImage = 'image-to-image',
|
||||
ImageToText = 'image-to-text',
|
||||
}
|
||||
|
||||
const CopilotProviderOptionsSchema = z.object({
|
||||
signal: z.instanceof(AbortSignal).optional(),
|
||||
user: z.string().optional(),
|
||||
});
|
||||
|
||||
const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
PromptConfigStrictSchema
|
||||
).optional();
|
||||
|
||||
export type CopilotChatOptions = z.infer<typeof CopilotChatOptionsSchema>;
|
||||
|
||||
const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({
|
||||
dimensions: z.number(),
|
||||
}).optional();
|
||||
|
||||
export type CopilotEmbeddingOptions = z.infer<
|
||||
typeof CopilotEmbeddingOptionsSchema
|
||||
>;
|
||||
|
||||
const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge(
|
||||
PromptConfigStrictSchema
|
||||
)
|
||||
.extend({
|
||||
seed: z.number().optional(),
|
||||
})
|
||||
.optional();
|
||||
|
||||
export type CopilotImageOptions = z.infer<typeof CopilotImageOptionsSchema>;
|
||||
|
||||
export type CopilotContextFile = {
|
||||
id: string; // fileId
|
||||
created_at: number;
|
||||
// embedding status
|
||||
status: 'in_progress' | 'completed' | 'failed';
|
||||
};
|
||||
|
||||
export interface CopilotProvider {
|
||||
readonly type: CopilotProviderType;
|
||||
getCapabilities(): CopilotCapability[];
|
||||
isModelAvailable(model: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface CopilotTextToTextProvider extends CopilotProvider {
|
||||
generateText(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
generateTextStream(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export interface CopilotTextToEmbeddingProvider extends CopilotProvider {
|
||||
generateEmbedding(
|
||||
messages: string[] | string,
|
||||
model: string,
|
||||
options?: CopilotEmbeddingOptions
|
||||
): Promise<number[][]>;
|
||||
}
|
||||
|
||||
export interface CopilotTextToImageProvider extends CopilotProvider {
|
||||
generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): Promise<Array<string>>;
|
||||
generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotImageOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export interface CopilotImageToTextProvider extends CopilotProvider {
|
||||
generateText(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotChatOptions
|
||||
): Promise<string>;
|
||||
generateTextStream(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotChatOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export interface CopilotImageToImageProvider extends CopilotProvider {
|
||||
generateImages(
|
||||
messages: PromptMessage[],
|
||||
model: string,
|
||||
options?: CopilotImageOptions
|
||||
): Promise<Array<string>>;
|
||||
generateImagesStream(
|
||||
messages: PromptMessage[],
|
||||
model?: string,
|
||||
options?: CopilotImageOptions
|
||||
): AsyncIterable<string>;
|
||||
}
|
||||
|
||||
export type CapabilityToCopilotProvider = {
|
||||
[CopilotCapability.TextToText]: CopilotTextToTextProvider;
|
||||
[CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider;
|
||||
[CopilotCapability.TextToImage]: CopilotTextToImageProvider;
|
||||
[CopilotCapability.ImageToText]: CopilotImageToTextProvider;
|
||||
[CopilotCapability.ImageToImage]: CopilotImageToImageProvider;
|
||||
};
|
||||
|
||||
export type CopilotTextProvider =
|
||||
| CopilotTextToTextProvider
|
||||
| CopilotImageToTextProvider;
|
||||
export type CopilotImageProvider =
|
||||
| CopilotTextToImageProvider
|
||||
| CopilotImageToImageProvider;
|
||||
export type CopilotAllProvider =
|
||||
| CopilotTextProvider
|
||||
| CopilotImageProvider
|
||||
| CopilotTextToEmbeddingProvider;
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { ChatPrompt, PromptService } from '../../prompt';
|
||||
import { CopilotProviderService } from '../../providers';
|
||||
import { CopilotChatOptions, CopilotImageProvider } from '../../types';
|
||||
import {
|
||||
CopilotChatOptions,
|
||||
CopilotImageProvider,
|
||||
CopilotProviderFactory,
|
||||
} from '../../providers';
|
||||
import { WorkflowNodeData, WorkflowNodeType } from '../types';
|
||||
import { NodeExecuteResult, NodeExecuteState, NodeExecutorType } from './types';
|
||||
import { AutoRegisteredWorkflowExecutor } from './utils';
|
||||
@@ -11,7 +14,7 @@ import { AutoRegisteredWorkflowExecutor } from './utils';
|
||||
export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor {
|
||||
constructor(
|
||||
private readonly promptService: PromptService,
|
||||
private readonly providerService: CopilotProviderService
|
||||
private readonly providerFactory: CopilotProviderFactory
|
||||
) {
|
||||
super();
|
||||
}
|
||||
@@ -42,7 +45,7 @@ export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor {
|
||||
`Prompt ${data.promptName} not found when running workflow node ${data.name}`
|
||||
);
|
||||
}
|
||||
const provider = await this.providerService.getProviderByModel(
|
||||
const provider = await this.providerFactory.getProviderByModel(
|
||||
prompt.model
|
||||
);
|
||||
if (provider && 'generateImages' in provider) {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { ChatPrompt, PromptService } from '../../prompt';
|
||||
import { CopilotProviderService } from '../../providers';
|
||||
import { CopilotChatOptions, CopilotTextProvider } from '../../types';
|
||||
import {
|
||||
CopilotChatOptions,
|
||||
CopilotProviderFactory,
|
||||
CopilotTextProvider,
|
||||
} from '../../providers';
|
||||
import { WorkflowNodeData, WorkflowNodeType } from '../types';
|
||||
import { NodeExecuteResult, NodeExecuteState, NodeExecutorType } from './types';
|
||||
import { AutoRegisteredWorkflowExecutor } from './utils';
|
||||
@@ -11,7 +14,7 @@ import { AutoRegisteredWorkflowExecutor } from './utils';
|
||||
export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor {
|
||||
constructor(
|
||||
private readonly promptService: PromptService,
|
||||
private readonly providerService: CopilotProviderService
|
||||
private readonly providerFactory: CopilotProviderFactory
|
||||
) {
|
||||
super();
|
||||
}
|
||||
@@ -42,7 +45,7 @@ export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor {
|
||||
`Prompt ${data.promptName} not found when running workflow node ${data.name}`
|
||||
);
|
||||
}
|
||||
const provider = await this.providerService.getProviderByModel(
|
||||
const provider = await this.providerFactory.getProviderByModel(
|
||||
prompt.model
|
||||
);
|
||||
if (provider && 'generateText' in provider) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { CopilotChatOptions } from '../../types';
|
||||
import { CopilotChatOptions } from '../../providers';
|
||||
import type { WorkflowNode } from '../node';
|
||||
import { WorkflowNodeData, WorkflowParams } from '../types';
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import Piscina from 'piscina';
|
||||
|
||||
import { CopilotChatOptions } from '../types';
|
||||
import { CopilotChatOptions } from '../providers';
|
||||
import type { NodeExecuteResult, NodeExecutor } from './executor';
|
||||
import { getWorkflowExecutor, NodeExecuteState } from './executor';
|
||||
import type {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { CopilotChatOptions } from '../types';
|
||||
import { CopilotChatOptions } from '../providers';
|
||||
import { WorkflowGraphList } from './graph';
|
||||
import { WorkflowNode } from './node';
|
||||
import type { WorkflowGraph, WorkflowGraphInstances } from './types';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import { CopilotChatOptions } from '../types';
|
||||
import { CopilotChatOptions } from '../providers';
|
||||
import { NodeExecuteState } from './executor';
|
||||
import { WorkflowNode } from './node';
|
||||
import type { WorkflowGraphInstances, WorkflowNodeState } from './types';
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { defineModuleConfig } from '../../base';
|
||||
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
customerIo: {
|
||||
enabled: boolean;
|
||||
token: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
defineModuleConfig('customerIo', {
|
||||
enabled: {
|
||||
desc: 'Enable customer.io integration',
|
||||
default: false,
|
||||
},
|
||||
token: {
|
||||
desc: 'Customer.io token',
|
||||
default: '',
|
||||
schema: { type: 'string' },
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
import './config';
|
||||
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { CustomerIoService } from './service';
|
||||
|
||||
@Module({
|
||||
providers: [CustomerIoService],
|
||||
})
|
||||
export class CustomerIoModule {}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config, OnEvent } from '../../base';
|
||||
|
||||
@Injectable()
|
||||
export class CustomerIoService {
|
||||
#fetch: ((url: string, options?: RequestInit) => Promise<Response>) | null =
|
||||
null;
|
||||
constructor(private readonly config: Config) {}
|
||||
|
||||
@OnEvent('config.init')
|
||||
setup() {
|
||||
const { enabled, token } = this.config.customerIo;
|
||||
if (enabled && token) {
|
||||
this.#fetch = (url, options) => {
|
||||
return fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
Authorization: `Basic ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
...options?.headers,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else {
|
||||
this.#fetch = null;
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if (event.updates.customerIo) {
|
||||
this.setup();
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent('user.created')
|
||||
@OnEvent('user.updated')
|
||||
async onUserUpdated(user: Events['user.updated'] | Events['user.created']) {
|
||||
await this.#fetch?.(
|
||||
`https://track.customer.io/api/v1/customers/${user.id}`,
|
||||
{
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({
|
||||
name: user.name,
|
||||
email: user.email,
|
||||
created_at: Number(user.createdAt) / 1000,
|
||||
}),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@OnEvent('user.deleted')
|
||||
async onUserDeleted(user: Events['user.deleted']) {
|
||||
if (user.emailVerifiedAt) {
|
||||
// suppress email if email is verified
|
||||
await this.#fetch?.(
|
||||
`https://track.customer.io/api/v1/customers/${user.email}/suppress`,
|
||||
{
|
||||
method: 'POST',
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
await this.#fetch?.(
|
||||
`https://track.customer.io/api/v1/customers/${user.id}`,
|
||||
{
|
||||
method: 'DELETE',
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
import { defineStartupConfig, ModuleConfig } from '../../base/config';
|
||||
|
||||
export interface GCloudConfig {
|
||||
enabled: boolean;
|
||||
}
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
gcloud: ModuleConfig<GCloudConfig>;
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.gcloud', {
|
||||
enabled: false,
|
||||
});
|
||||
@@ -1,14 +1,10 @@
|
||||
import './config';
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { Global } from '@nestjs/common';
|
||||
|
||||
import { Plugin } from '../registry';
|
||||
import { GCloudLogging } from './logging';
|
||||
import { GCloudMetrics } from './metrics';
|
||||
|
||||
@Global()
|
||||
@Plugin({
|
||||
name: 'gcloud',
|
||||
@Module({
|
||||
imports: [GCloudMetrics, GCloudLogging],
|
||||
})
|
||||
export class GCloudModule {}
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import { Global } from '@nestjs/common';
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
|
||||
import { OptionalModule } from '../../../base';
|
||||
import { loggerProvider } from './service';
|
||||
import { LoggerProvider } from './service';
|
||||
|
||||
@Global()
|
||||
@OptionalModule({
|
||||
if: config => config.metrics.enabled,
|
||||
overrides: [loggerProvider],
|
||||
@Module({
|
||||
providers: [LoggerProvider],
|
||||
exports: [LoggerProvider],
|
||||
})
|
||||
export class GCloudLogging {}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ const moreMetadata = format(info => {
|
||||
return info;
|
||||
});
|
||||
|
||||
export const loggerProvider: Provider<LoggerService> = {
|
||||
export const LoggerProvider: Provider<LoggerService> = {
|
||||
provide: LoggerProvide,
|
||||
useFactory: () => {
|
||||
const instance = createLogger({
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter';
|
||||
import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
|
||||
import { GcpDetectorSync } from '@google-cloud/opentelemetry-resource-util';
|
||||
import { Global, Provider } from '@nestjs/common';
|
||||
import { Global, Injectable, Module, Provider } from '@nestjs/common';
|
||||
import { getEnv } from '@opentelemetry/core';
|
||||
import { Resource } from '@opentelemetry/resources';
|
||||
import {
|
||||
@@ -14,9 +14,9 @@ import {
|
||||
SEMRESATTRS_K8S_POD_NAME,
|
||||
} from '@opentelemetry/semantic-conventions';
|
||||
|
||||
import { OptionalModule } from '../../base';
|
||||
import { OpentelemetryFactory } from '../../base/metrics';
|
||||
|
||||
@Injectable()
|
||||
export class GCloudOpentelemetryFactory extends OpentelemetryFactory {
|
||||
override getResource(): Resource {
|
||||
const env = getEnv();
|
||||
@@ -47,14 +47,14 @@ export class GCloudOpentelemetryFactory extends OpentelemetryFactory {
|
||||
}
|
||||
}
|
||||
|
||||
const factorProvider: Provider = {
|
||||
const FactorProvider: Provider = {
|
||||
provide: OpentelemetryFactory,
|
||||
useFactory: () => new GCloudOpentelemetryFactory(),
|
||||
useClass: GCloudOpentelemetryFactory,
|
||||
};
|
||||
|
||||
@Global()
|
||||
@OptionalModule({
|
||||
if: config => config.metrics.enabled,
|
||||
overrides: [factorProvider],
|
||||
@Module({
|
||||
providers: [FactorProvider],
|
||||
exports: [FactorProvider],
|
||||
})
|
||||
export class GCloudMetrics {}
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
import './captcha';
|
||||
import './copilot';
|
||||
import './gcloud';
|
||||
import './oauth';
|
||||
import './payment';
|
||||
import './storage';
|
||||
import './worker';
|
||||
|
||||
export {
|
||||
enablePlugin,
|
||||
REGISTERED_PLUGINS,
|
||||
ENABLED_PLUGINS as USED_PLUGINS,
|
||||
} from './registry';
|
||||
@@ -1,10 +1,11 @@
|
||||
import { OptionalModule } from '../../base';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { PermissionModule } from '../../core/permission';
|
||||
import { QuotaModule } from '../../core/quota';
|
||||
import { LicenseResolver } from './resolver';
|
||||
import { LicenseService } from './service';
|
||||
|
||||
@OptionalModule({
|
||||
@Module({
|
||||
imports: [QuotaModule, PermissionModule],
|
||||
providers: [LicenseService, LicenseResolver],
|
||||
})
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
Resolver,
|
||||
} from '@nestjs/graphql';
|
||||
|
||||
import { ActionForbidden, Config } from '../../base';
|
||||
import { UseNamedGuard } from '../../base';
|
||||
import { CurrentUser } from '../../core/auth';
|
||||
import { AccessController } from '../../core/permission';
|
||||
import { WorkspaceType } from '../../core/workspaces';
|
||||
@@ -34,10 +34,10 @@ export class License {
|
||||
expiredAt!: Date | null;
|
||||
}
|
||||
|
||||
@UseNamedGuard('selfhost')
|
||||
@Resolver(() => WorkspaceType)
|
||||
export class LicenseResolver {
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly service: LicenseService,
|
||||
private readonly ac: AccessController
|
||||
) {}
|
||||
@@ -51,13 +51,6 @@ export class LicenseResolver {
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Parent() workspace: WorkspaceType
|
||||
): Promise<License | null> {
|
||||
// NOTE(@forehalo):
|
||||
// we can't simply disable license resolver for non-selfhosted server
|
||||
// it will make the gql codegen messed up.
|
||||
if (!this.config.isSelfhosted) {
|
||||
return null;
|
||||
}
|
||||
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspace.id)
|
||||
@@ -71,10 +64,6 @@ export class LicenseResolver {
|
||||
@Args('workspaceId') workspaceId: string,
|
||||
@Args('license') license: string
|
||||
) {
|
||||
if (!this.config.isSelfhosted) {
|
||||
throw new ActionForbidden();
|
||||
}
|
||||
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
@@ -88,10 +77,6 @@ export class LicenseResolver {
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string
|
||||
) {
|
||||
if (!this.config.isSelfhosted) {
|
||||
throw new ActionForbidden();
|
||||
}
|
||||
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
@@ -105,10 +90,6 @@ export class LicenseResolver {
|
||||
@CurrentUser() user: CurrentUser,
|
||||
@Args('workspaceId') workspaceId: string
|
||||
) {
|
||||
if (!this.config.isSelfhosted) {
|
||||
throw new ActionForbidden();
|
||||
}
|
||||
|
||||
await this.ac
|
||||
.user(user.id)
|
||||
.workspace(workspaceId)
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { InstalledLicense, PrismaClient } from '@prisma/client';
|
||||
|
||||
import {
|
||||
Config,
|
||||
EventBus,
|
||||
InternalServerError,
|
||||
LicenseNotFound,
|
||||
@@ -22,35 +21,22 @@ interface License {
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class LicenseService implements OnModuleInit {
|
||||
export class LicenseService {
|
||||
private readonly logger = new Logger(LicenseService.name);
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly db: PrismaClient,
|
||||
private readonly event: EventBus,
|
||||
private readonly models: Models
|
||||
) {}
|
||||
|
||||
async onModuleInit() {
|
||||
if (this.config.isSelfhosted) {
|
||||
this.event.on(
|
||||
'workspace.subscription.activated',
|
||||
this.onWorkspaceSubscriptionUpdated
|
||||
);
|
||||
this.event.on(
|
||||
'workspace.subscription.canceled',
|
||||
this.onWorkspaceSubscriptionCanceled
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private readonly onWorkspaceSubscriptionUpdated = async ({
|
||||
@OnEvent('workspace.subscription.activated')
|
||||
async onWorkspaceSubscriptionUpdated({
|
||||
workspaceId,
|
||||
plan,
|
||||
recurring,
|
||||
quantity,
|
||||
}: Events['workspace.subscription.activated']) => {
|
||||
}: Events['workspace.subscription.activated']) {
|
||||
switch (plan) {
|
||||
case SubscriptionPlan.SelfHostedTeam:
|
||||
await this.models.workspaceFeature.add(
|
||||
@@ -66,12 +52,13 @@ export class LicenseService implements OnModuleInit {
|
||||
default:
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private readonly onWorkspaceSubscriptionCanceled = async ({
|
||||
@OnEvent('workspace.subscription.canceled')
|
||||
async onWorkspaceSubscriptionCanceled({
|
||||
workspaceId,
|
||||
plan,
|
||||
}: Events['workspace.subscription.canceled']) => {
|
||||
}: Events['workspace.subscription.canceled']) {
|
||||
switch (plan) {
|
||||
case SubscriptionPlan.SelfHostedTeam:
|
||||
await this.models.workspaceFeature.remove(workspaceId, 'team_plan_v1');
|
||||
@@ -79,7 +66,7 @@ export class LicenseService implements OnModuleInit {
|
||||
default:
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async getLicense(workspaceId: string) {
|
||||
return this.db.installedLicense.findUnique({
|
||||
@@ -201,10 +188,6 @@ export class LicenseService implements OnModuleInit {
|
||||
|
||||
@OnEvent('workspace.members.updated')
|
||||
async updateTeamSeats(payload: Events['workspace.members.updated']) {
|
||||
if (!this.config.isSelfhosted) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { workspaceId, count } = payload;
|
||||
|
||||
const license = await this.db.installedLicense.findUnique({
|
||||
@@ -251,12 +234,8 @@ export class LicenseService implements OnModuleInit {
|
||||
throw new Error('Timeout checking seat update result.');
|
||||
}
|
||||
|
||||
@Cron(CronExpression.EVERY_10_MINUTES)
|
||||
@Cron(CronExpression.EVERY_10_MINUTES, { disabled: !env.selfhosted })
|
||||
async licensesHealthCheck() {
|
||||
if (!this.config.isSelfhosted) {
|
||||
return;
|
||||
}
|
||||
|
||||
const licenses = await this.db.installedLicense.findMany({
|
||||
where: {
|
||||
validatedAt: {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { defineStartupConfig, ModuleConfig } from '../../base/config';
|
||||
import { defineModuleConfig, JSONSchema } from '../../base';
|
||||
|
||||
export interface OAuthProviderConfig {
|
||||
clientId: string;
|
||||
@@ -23,23 +23,53 @@ export enum OAuthProviderName {
|
||||
GitHub = 'github',
|
||||
OIDC = 'oidc',
|
||||
}
|
||||
|
||||
type OAuthProviderConfigMapping = {
|
||||
[OAuthProviderName.Google]: OAuthProviderConfig;
|
||||
[OAuthProviderName.GitHub]: OAuthProviderConfig;
|
||||
[OAuthProviderName.OIDC]: OAuthOIDCProviderConfig;
|
||||
};
|
||||
|
||||
export interface OAuthConfig {
|
||||
providers: Partial<OAuthProviderConfigMapping>;
|
||||
}
|
||||
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
oauth: ModuleConfig<OAuthConfig>;
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
oauth: {
|
||||
providers: {
|
||||
[OAuthProviderName.Google]: ConfigItem<OAuthProviderConfig>;
|
||||
[OAuthProviderName.GitHub]: ConfigItem<OAuthProviderConfig>;
|
||||
[OAuthProviderName.OIDC]: ConfigItem<OAuthOIDCProviderConfig>;
|
||||
};
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.oauth', {
|
||||
providers: {},
|
||||
const schema: JSONSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
clientId: { type: 'string' },
|
||||
clientSecret: { type: 'string' },
|
||||
args: { type: 'object' },
|
||||
},
|
||||
};
|
||||
|
||||
defineModuleConfig('oauth', {
|
||||
'providers.google': {
|
||||
desc: 'Google OAuth provider config',
|
||||
default: {
|
||||
clientId: '',
|
||||
clientSecret: '',
|
||||
},
|
||||
schema,
|
||||
link: 'https://developers.google.com/identity/protocols/oauth2/web-server',
|
||||
},
|
||||
'providers.github': {
|
||||
desc: 'GitHub OAuth provider config',
|
||||
default: {
|
||||
clientId: '',
|
||||
clientSecret: '',
|
||||
},
|
||||
schema,
|
||||
link: 'https://docs.github.com/en/apps/oauth-apps',
|
||||
},
|
||||
'providers.oidc': {
|
||||
desc: 'OIDC OAuth provider config',
|
||||
default: {
|
||||
clientId: '',
|
||||
clientSecret: '',
|
||||
issuer: '',
|
||||
args: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -24,8 +24,8 @@ import {
|
||||
import { AuthService, Public } from '../../core/auth';
|
||||
import { Models } from '../../models';
|
||||
import { OAuthProviderName } from './config';
|
||||
import { OAuthProviderFactory } from './factory';
|
||||
import { OAuthAccount, Tokens } from './providers/def';
|
||||
import { OAuthProviderFactory } from './register';
|
||||
import { OAuthService } from './service';
|
||||
|
||||
@Controller('/api/oauth')
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { ServerFeature, ServerService } from '../../core';
|
||||
import { OAuthProviderName } from './config';
|
||||
import type { OAuthProvider } from './providers/def';
|
||||
|
||||
@Injectable()
|
||||
export class OAuthProviderFactory {
|
||||
constructor(private readonly server: ServerService) {}
|
||||
|
||||
private readonly logger = new Logger(OAuthProviderFactory.name);
|
||||
readonly #providers = new Map<OAuthProviderName, OAuthProvider>();
|
||||
|
||||
get providers() {
|
||||
return Array.from(this.#providers.keys());
|
||||
}
|
||||
|
||||
get(name: OAuthProviderName): OAuthProvider | undefined {
|
||||
return this.#providers.get(name);
|
||||
}
|
||||
|
||||
register(provider: OAuthProvider) {
|
||||
this.#providers.set(provider.provider, provider);
|
||||
this.logger.log(`OAuth provider [${provider.provider}] registered.`);
|
||||
this.server.enableFeature(ServerFeature.OAuth);
|
||||
}
|
||||
|
||||
unregister(provider: OAuthProvider) {
|
||||
this.#providers.delete(provider.provider);
|
||||
this.logger.log(`OAuth provider [${provider.provider}] unregistered.`);
|
||||
if (this.#providers.size === 0) {
|
||||
this.server.disableFeature(ServerFeature.OAuth);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,18 +1,18 @@
|
||||
import './config';
|
||||
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ServerConfigModule } from '../../core';
|
||||
import { AuthModule } from '../../core/auth';
|
||||
import { ServerFeature } from '../../core/config';
|
||||
import { UserModule } from '../../core/user';
|
||||
import { Plugin } from '../registry';
|
||||
import { OAuthController } from './controller';
|
||||
import { OAuthProviderFactory } from './factory';
|
||||
import { OAuthProviders } from './providers';
|
||||
import { OAuthProviderFactory } from './register';
|
||||
import { OAuthResolver } from './resolver';
|
||||
import { OAuthService } from './service';
|
||||
|
||||
@Plugin({
|
||||
name: 'oauth',
|
||||
imports: [AuthModule, UserModule],
|
||||
@Module({
|
||||
imports: [AuthModule, UserModule, ServerConfigModule],
|
||||
providers: [
|
||||
OAuthProviderFactory,
|
||||
OAuthService,
|
||||
@@ -20,7 +20,5 @@ import { OAuthService } from './service';
|
||||
...OAuthProviders,
|
||||
],
|
||||
controllers: [OAuthController],
|
||||
contributesTo: ServerFeature.OAuth,
|
||||
if: config => config.flavor.graphql && !!config.plugins.oauth,
|
||||
})
|
||||
export class OAuthModule {}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
import { Config, OnEvent } from '../../../base';
|
||||
import { OAuthProviderName } from '../config';
|
||||
import { OAuthProviderFactory } from '../factory';
|
||||
|
||||
export interface OAuthAccount {
|
||||
id: string;
|
||||
@@ -13,9 +17,42 @@ export interface Tokens {
|
||||
expiresAt?: Date;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export abstract class OAuthProvider {
|
||||
abstract provider: OAuthProviderName;
|
||||
abstract getAuthUrl(state: string): string;
|
||||
abstract getToken(code: string): Promise<Tokens>;
|
||||
abstract getUser(token: string): Promise<OAuthAccount>;
|
||||
|
||||
protected readonly logger = new Logger(this.constructor.name);
|
||||
@Inject() private readonly factory!: OAuthProviderFactory;
|
||||
@Inject() private readonly AFFiNEConfig!: Config;
|
||||
|
||||
get config() {
|
||||
return this.AFFiNEConfig.oauth.providers[this.provider];
|
||||
}
|
||||
|
||||
get configured() {
|
||||
return this.config && this.config.clientId && this.config.clientSecret;
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
onConfigInit() {
|
||||
this.setup();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigUpdated(event: Events['config.changed']) {
|
||||
if ('oauth' in event.updates) {
|
||||
this.setup();
|
||||
}
|
||||
}
|
||||
|
||||
protected setup() {
|
||||
if (this.configured) {
|
||||
this.factory.register(this);
|
||||
} else {
|
||||
this.factory.unregister(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config, InvalidOauthCallbackCode, URLHelper } from '../../../base';
|
||||
import { InvalidOauthCallbackCode, URLHelper } from '../../../base';
|
||||
import { OAuthProviderName } from '../config';
|
||||
import { AutoRegisteredOAuthProvider } from '../register';
|
||||
import { OAuthProvider } from './def';
|
||||
|
||||
interface AuthTokenResponse {
|
||||
access_token: string;
|
||||
@@ -18,13 +18,10 @@ export interface UserInfo {
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class GithubOAuthProvider extends AutoRegisteredOAuthProvider {
|
||||
export class GithubOAuthProvider extends OAuthProvider {
|
||||
provider = OAuthProviderName.GitHub;
|
||||
|
||||
constructor(
|
||||
protected readonly AFFiNEConfig: Config,
|
||||
private readonly url: URLHelper
|
||||
) {
|
||||
constructor(private readonly url: URLHelper) {
|
||||
super();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config, InvalidOauthCallbackCode, URLHelper } from '../../../base';
|
||||
import { InvalidOauthCallbackCode, URLHelper } from '../../../base';
|
||||
import { OAuthProviderName } from '../config';
|
||||
import { AutoRegisteredOAuthProvider } from '../register';
|
||||
import { OAuthProvider } from './def';
|
||||
|
||||
interface GoogleOAuthTokenResponse {
|
||||
access_token: string;
|
||||
@@ -20,13 +20,10 @@ export interface UserInfo {
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class GoogleOAuthProvider extends AutoRegisteredOAuthProvider {
|
||||
export class GoogleOAuthProvider extends OAuthProvider {
|
||||
override provider = OAuthProviderName.Google;
|
||||
|
||||
constructor(
|
||||
protected readonly AFFiNEConfig: Config,
|
||||
private readonly url: URLHelper
|
||||
) {
|
||||
constructor(private readonly url: URLHelper) {
|
||||
super();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { Config, URLHelper } from '../../../base';
|
||||
import { URLHelper } from '../../../base';
|
||||
import {
|
||||
OAuthOIDCProviderConfig,
|
||||
OAuthProviderName,
|
||||
OIDCArgs,
|
||||
} from '../config';
|
||||
import { AutoRegisteredOAuthProvider } from '../register';
|
||||
import { OAuthAccount, Tokens } from './def';
|
||||
import { OAuthAccount, OAuthProvider, Tokens } from './def';
|
||||
|
||||
const OIDCTokenSchema = z.object({
|
||||
access_token: z.string(),
|
||||
@@ -163,26 +162,26 @@ class OIDCClient {
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class OIDCProvider
|
||||
extends AutoRegisteredOAuthProvider
|
||||
implements OnModuleInit
|
||||
{
|
||||
export class OIDCProvider extends OAuthProvider {
|
||||
override provider = OAuthProviderName.OIDC;
|
||||
private client: OIDCClient | null = null;
|
||||
|
||||
constructor(
|
||||
protected readonly AFFiNEConfig: Config,
|
||||
private readonly url: URLHelper
|
||||
) {
|
||||
constructor(private readonly url: URLHelper) {
|
||||
super();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-misused-promises
|
||||
override async onModuleInit() {
|
||||
const config = this.optionalConfig as OAuthOIDCProviderConfig;
|
||||
if (config && config.issuer && config.clientId && config.clientSecret) {
|
||||
this.client = await OIDCClient.create(config, this.url);
|
||||
super.onModuleInit();
|
||||
protected override setup() {
|
||||
super.setup();
|
||||
if (this.configured) {
|
||||
OIDCClient.create(this.config as OAuthOIDCProviderConfig, this.url)
|
||||
.then(client => {
|
||||
this.client = client;
|
||||
})
|
||||
.catch(e => {
|
||||
this.logger.error('Failed to create OIDC client', e);
|
||||
});
|
||||
} else {
|
||||
this.client = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
|
||||
import { Config } from '../../base';
|
||||
import { OAuthProviderName } from './config';
|
||||
import { OAuthProvider } from './providers/def';
|
||||
|
||||
const PROVIDERS: Map<OAuthProviderName, OAuthProvider> = new Map();
|
||||
|
||||
export function registerOAuthProvider(
|
||||
name: OAuthProviderName,
|
||||
provider: OAuthProvider
|
||||
) {
|
||||
PROVIDERS.set(name, provider);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class OAuthProviderFactory {
|
||||
get providers() {
|
||||
return Array.from(PROVIDERS.keys());
|
||||
}
|
||||
|
||||
get(name: OAuthProviderName): OAuthProvider | undefined {
|
||||
return PROVIDERS.get(name);
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class AutoRegisteredOAuthProvider
|
||||
extends OAuthProvider
|
||||
implements OnModuleInit
|
||||
{
|
||||
protected abstract AFFiNEConfig: Config;
|
||||
|
||||
get optionalConfig() {
|
||||
return this.AFFiNEConfig.plugins.oauth?.providers?.[this.provider];
|
||||
}
|
||||
|
||||
get config() {
|
||||
const config = this.optionalConfig;
|
||||
|
||||
if (!config) {
|
||||
throw new Error(
|
||||
`OAuthProvider Config should not be used before registered`
|
||||
);
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
const config = this.optionalConfig;
|
||||
if (config && config.clientId && config.clientSecret) {
|
||||
registerOAuthProvider(this.provider, this);
|
||||
new Logger(`OAuthProvider:${this.provider}`).log(
|
||||
'OAuth provider registered.'
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import { registerEnumType, ResolveField, Resolver } from '@nestjs/graphql';
|
||||
|
||||
import { ServerConfigType } from '../../core/config/types';
|
||||
import { OAuthProviderName } from './config';
|
||||
import { OAuthProviderFactory } from './register';
|
||||
import { OAuthProviderFactory } from './factory';
|
||||
|
||||
registerEnumType(OAuthProviderName, { name: 'OAuthProviderType' });
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { SessionCache } from '../../base';
|
||||
import { OAuthProviderName } from './config';
|
||||
import { OAuthProviderFactory } from './register';
|
||||
import { OAuthProviderFactory } from './factory';
|
||||
|
||||
const OAUTH_STATE_KEY = 'OAUTH_STATE';
|
||||
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
import type { Stripe } from 'stripe';
|
||||
|
||||
import {
|
||||
defineRuntimeConfig,
|
||||
defineStartupConfig,
|
||||
ModuleConfig,
|
||||
} from '../../base/config';
|
||||
import { defineModuleConfig } from '../../base';
|
||||
|
||||
export interface PaymentStartupConfig {
|
||||
stripe?: {
|
||||
@@ -19,16 +15,40 @@ export interface PaymentRuntimeConfig {
|
||||
showLifetimePrice: boolean;
|
||||
}
|
||||
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
payment: ModuleConfig<PaymentStartupConfig, PaymentRuntimeConfig>;
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
payment: {
|
||||
enabled: boolean;
|
||||
showLifetimePrice: boolean;
|
||||
apiKey: string;
|
||||
webhookKey: string;
|
||||
stripe: ConfigItem<{} & Stripe.StripeConfig>;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.payment', {});
|
||||
defineRuntimeConfig('plugins.payment', {
|
||||
defineModuleConfig('payment', {
|
||||
enabled: {
|
||||
desc: 'Whether enable payment plugin',
|
||||
default: false,
|
||||
},
|
||||
showLifetimePrice: {
|
||||
desc: 'Whether enable lifetime price and allow user to pay for it.',
|
||||
default: true,
|
||||
},
|
||||
apiKey: {
|
||||
desc: 'Stripe API key to enable payment service.',
|
||||
default: '',
|
||||
env: 'STRIPE_API_KEY',
|
||||
},
|
||||
webhookKey: {
|
||||
desc: 'Stripe webhook key to enable payment service.',
|
||||
default: '',
|
||||
env: 'STRIPE_WEBHOOK_KEY',
|
||||
},
|
||||
stripe: {
|
||||
desc: 'Stripe API keys',
|
||||
default: {},
|
||||
link: 'https://docs.stripe.com/api',
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,37 +1,32 @@
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type { RawBodyRequest } from '@nestjs/common';
|
||||
import { Controller, Logger, Post, Req } from '@nestjs/common';
|
||||
import type { Request } from 'express';
|
||||
import Stripe from 'stripe';
|
||||
|
||||
import { Config, EventBus, InternalServerError } from '../../base';
|
||||
import { Public } from '../../core/auth';
|
||||
import { StripeFactory } from './stripe';
|
||||
|
||||
@Controller('/api/stripe')
|
||||
export class StripeWebhookController {
|
||||
private readonly webhookKey: string;
|
||||
private readonly logger = new Logger(StripeWebhookController.name);
|
||||
|
||||
constructor(
|
||||
config: Config,
|
||||
private readonly stripe: Stripe,
|
||||
private readonly config: Config,
|
||||
private readonly stripeProvider: StripeFactory,
|
||||
private readonly event: EventBus
|
||||
) {
|
||||
assert(config.plugins.payment.stripe);
|
||||
this.webhookKey = config.plugins.payment.stripe.keys.webhookKey;
|
||||
}
|
||||
) {}
|
||||
|
||||
@Public()
|
||||
@Post('/webhook')
|
||||
async handleWebhook(@Req() req: RawBodyRequest<Request>) {
|
||||
const webhookKey = this.config.payment.webhookKey;
|
||||
// Retrieve the event by verifying the signature using the raw body and secret.
|
||||
const signature = req.headers['stripe-signature'];
|
||||
try {
|
||||
const event = this.stripe.webhooks.constructEvent(
|
||||
const event = this.stripeProvider.stripe.webhooks.constructEvent(
|
||||
req.rawBody ?? '',
|
||||
signature ?? '',
|
||||
this.webhookKey
|
||||
webhookKey
|
||||
);
|
||||
|
||||
this.logger.debug(
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import './config';
|
||||
|
||||
import { ServerFeature } from '../../core/config';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ServerConfigModule } from '../../core';
|
||||
import { FeatureModule } from '../../core/features';
|
||||
import { MailModule } from '../../core/mail';
|
||||
import { PermissionModule } from '../../core/permission';
|
||||
import { QuotaModule } from '../../core/quota';
|
||||
import { UserModule } from '../../core/user';
|
||||
import { WorkspaceModule } from '../../core/workspaces';
|
||||
import { Plugin } from '../registry';
|
||||
import { StripeWebhookController } from './controller';
|
||||
import { SubscriptionCronJobs } from './cron';
|
||||
import { LicenseController } from './license/controller';
|
||||
@@ -23,11 +24,10 @@ import {
|
||||
WorkspaceSubscriptionResolver,
|
||||
} from './resolver';
|
||||
import { SubscriptionService } from './service';
|
||||
import { StripeProvider } from './stripe';
|
||||
import { StripeFactory, StripeProvider } from './stripe';
|
||||
import { StripeWebhook } from './webhook';
|
||||
|
||||
@Plugin({
|
||||
name: 'payment',
|
||||
@Module({
|
||||
imports: [
|
||||
FeatureModule,
|
||||
QuotaModule,
|
||||
@@ -35,8 +35,10 @@ import { StripeWebhook } from './webhook';
|
||||
PermissionModule,
|
||||
WorkspaceModule,
|
||||
MailModule,
|
||||
ServerConfigModule,
|
||||
],
|
||||
providers: [
|
||||
StripeFactory,
|
||||
StripeProvider,
|
||||
SubscriptionService,
|
||||
SubscriptionResolver,
|
||||
@@ -50,11 +52,5 @@ import { StripeWebhook } from './webhook';
|
||||
QuotaOverride,
|
||||
],
|
||||
controllers: [StripeWebhookController, LicenseController],
|
||||
requires: [
|
||||
'plugins.payment.stripe.keys.APIKey',
|
||||
'plugins.payment.stripe.keys.webhookKey',
|
||||
],
|
||||
contributesTo: ServerFeature.Payment,
|
||||
if: config => config.flavor.graphql,
|
||||
})
|
||||
export class PaymentModule {}
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
import { Public } from '../../../core/auth';
|
||||
import { SelfhostTeamSubscriptionManager } from '../manager/selfhost';
|
||||
import { SubscriptionService } from '../service';
|
||||
import { StripeFactory } from '../stripe';
|
||||
import {
|
||||
SubscriptionPlan,
|
||||
SubscriptionRecurring,
|
||||
@@ -53,7 +54,7 @@ export class LicenseController {
|
||||
private readonly mutex: Mutex,
|
||||
private readonly subscription: SubscriptionService,
|
||||
private readonly manager: SelfhostTeamSubscriptionManager,
|
||||
private readonly stripe: Stripe
|
||||
private readonly stripeProvider: StripeFactory
|
||||
) {}
|
||||
|
||||
@Post('/:license/activate')
|
||||
@@ -238,18 +239,20 @@ export class LicenseController {
|
||||
throw new LicenseNotFound();
|
||||
}
|
||||
|
||||
const subscriptionData = await this.stripe.subscriptions.retrieve(
|
||||
subscription.stripeSubscriptionId,
|
||||
{
|
||||
expand: ['customer'],
|
||||
}
|
||||
);
|
||||
const subscriptionData =
|
||||
await this.stripeProvider.stripe.subscriptions.retrieve(
|
||||
subscription.stripeSubscriptionId,
|
||||
{
|
||||
expand: ['customer'],
|
||||
}
|
||||
);
|
||||
|
||||
const customer = subscriptionData.customer as Stripe.Customer;
|
||||
try {
|
||||
const portal = await this.stripe.billingPortal.sessions.create({
|
||||
customer: customer.id,
|
||||
});
|
||||
const portal =
|
||||
await this.stripeProvider.stripe.billingPortal.sessions.create({
|
||||
customer: customer.id,
|
||||
});
|
||||
|
||||
return { url: portal.url };
|
||||
} catch (e) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { z } from 'zod';
|
||||
|
||||
import { UserNotFound } from '../../../base';
|
||||
import { ScheduleManager } from '../schedule';
|
||||
import { StripeFactory } from '../stripe';
|
||||
import {
|
||||
encodeLookupKey,
|
||||
KnownStripeInvoice,
|
||||
@@ -55,12 +56,16 @@ export const CheckoutParams = z.object({
|
||||
});
|
||||
|
||||
export abstract class SubscriptionManager {
|
||||
protected readonly scheduleManager = new ScheduleManager(this.stripe);
|
||||
protected readonly scheduleManager = new ScheduleManager(this.stripeProvider);
|
||||
constructor(
|
||||
protected readonly stripe: Stripe,
|
||||
protected readonly stripeProvider: StripeFactory,
|
||||
protected readonly db: PrismaClient
|
||||
) {}
|
||||
|
||||
get stripe() {
|
||||
return this.stripeProvider.stripe;
|
||||
}
|
||||
|
||||
abstract filterPrices(
|
||||
prices: KnownStripePrice[],
|
||||
customer?: UserStripeCustomer
|
||||
|
||||
@@ -3,11 +3,11 @@ import { randomUUID } from 'node:crypto';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PrismaClient, UserStripeCustomer } from '@prisma/client';
|
||||
import { pick } from 'lodash-es';
|
||||
import Stripe from 'stripe';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { SubscriptionPlanNotFound, URLHelper } from '../../../base';
|
||||
import { Mailer } from '../../../core/mail';
|
||||
import { StripeFactory } from '../stripe';
|
||||
import {
|
||||
KnownStripeInvoice,
|
||||
KnownStripePrice,
|
||||
@@ -43,12 +43,12 @@ export const SelfhostTeamSubscriptionIdentity = z.object({
|
||||
@Injectable()
|
||||
export class SelfhostTeamSubscriptionManager extends SubscriptionManager {
|
||||
constructor(
|
||||
stripe: Stripe,
|
||||
stripeProvider: StripeFactory,
|
||||
db: PrismaClient,
|
||||
private readonly url: URLHelper,
|
||||
private readonly mailer: Mailer
|
||||
) {
|
||||
super(stripe, db);
|
||||
super(stripeProvider, db);
|
||||
}
|
||||
|
||||
filterPrices(
|
||||
|
||||
@@ -5,17 +5,18 @@ import Stripe from 'stripe';
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
Config,
|
||||
EventBus,
|
||||
InternalServerError,
|
||||
InvalidCheckoutParameters,
|
||||
Mutex,
|
||||
Runtime,
|
||||
SubscriptionAlreadyExists,
|
||||
SubscriptionPlanNotFound,
|
||||
TooManyRequest,
|
||||
URLHelper,
|
||||
} from '../../../base';
|
||||
import { EarlyAccessType, FeatureService } from '../../../core/features';
|
||||
import { StripeFactory } from '../stripe';
|
||||
import {
|
||||
CouponType,
|
||||
KnownStripeInvoice,
|
||||
@@ -53,15 +54,15 @@ export const UserSubscriptionCheckoutArgs = z.object({
|
||||
@Injectable()
|
||||
export class UserSubscriptionManager extends SubscriptionManager {
|
||||
constructor(
|
||||
stripe: Stripe,
|
||||
stripeProvider: StripeFactory,
|
||||
db: PrismaClient,
|
||||
private readonly runtime: Runtime,
|
||||
private readonly config: Config,
|
||||
private readonly feature: FeatureService,
|
||||
private readonly event: EventBus,
|
||||
private readonly url: URLHelper,
|
||||
private readonly mutex: Mutex
|
||||
) {
|
||||
super(stripe, db);
|
||||
super(stripeProvider, db);
|
||||
}
|
||||
|
||||
async filterPrices(
|
||||
@@ -588,7 +589,7 @@ export class UserSubscriptionManager extends SubscriptionManager {
|
||||
{ proEarlyAccess, proSubscribed, onetime }: PriceStrategyStatus
|
||||
) {
|
||||
if (lookupKey.recurring === SubscriptionRecurring.Lifetime) {
|
||||
return this.runtime.fetch('plugins.payment/showLifetimePrice');
|
||||
return this.config.payment.showLifetimePrice;
|
||||
}
|
||||
|
||||
if (lookupKey.variant === SubscriptionVariant.Onetime) {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { PrismaClient, UserStripeCustomer } from '@prisma/client';
|
||||
import { omit, pick } from 'lodash-es';
|
||||
import Stripe from 'stripe';
|
||||
import { z } from 'zod';
|
||||
|
||||
import {
|
||||
@@ -12,6 +11,7 @@ import {
|
||||
URLHelper,
|
||||
} from '../../../base';
|
||||
import { Models } from '../../../models';
|
||||
import { StripeFactory } from '../stripe';
|
||||
import {
|
||||
KnownStripeInvoice,
|
||||
KnownStripePrice,
|
||||
@@ -46,13 +46,13 @@ export const WorkspaceSubscriptionCheckoutArgs = z.object({
|
||||
@Injectable()
|
||||
export class WorkspaceSubscriptionManager extends SubscriptionManager {
|
||||
constructor(
|
||||
stripe: Stripe,
|
||||
stripeProvider: StripeFactory,
|
||||
db: PrismaClient,
|
||||
private readonly url: URLHelper,
|
||||
private readonly event: EventBus,
|
||||
private readonly models: Models
|
||||
) {
|
||||
super(stripe, db);
|
||||
super(stripeProvider, db);
|
||||
}
|
||||
|
||||
filterPrices(
|
||||
|
||||
@@ -1,15 +1,24 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import Stripe from 'stripe';
|
||||
|
||||
import { StripeFactory } from './stripe';
|
||||
|
||||
@Injectable()
|
||||
export class ScheduleManager {
|
||||
private _schedule: Stripe.SubscriptionSchedule | null = null;
|
||||
private readonly logger = new Logger(ScheduleManager.name);
|
||||
|
||||
constructor(private readonly stripe: Stripe) {}
|
||||
constructor(private readonly stripeProvider: StripeFactory) {}
|
||||
|
||||
static create(stripe: Stripe, schedule?: Stripe.SubscriptionSchedule) {
|
||||
const manager = new ScheduleManager(stripe);
|
||||
get stripe() {
|
||||
return this.stripeProvider.stripe;
|
||||
}
|
||||
|
||||
static create(
|
||||
stripeProvider: StripeFactory,
|
||||
schedule?: Stripe.SubscriptionSchedule
|
||||
) {
|
||||
const manager = new ScheduleManager(stripeProvider);
|
||||
if (schedule) {
|
||||
manager._schedule = schedule;
|
||||
}
|
||||
@@ -56,9 +65,9 @@ export class ScheduleManager {
|
||||
return undefined;
|
||||
});
|
||||
|
||||
return ScheduleManager.create(this.stripe, s);
|
||||
return ScheduleManager.create(this.stripeProvider, s);
|
||||
} else {
|
||||
return ScheduleManager.create(this.stripe, schedule);
|
||||
return ScheduleManager.create(this.stripeProvider, schedule);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import type { User, UserStripeCustomer } from '@prisma/client';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import Stripe from 'stripe';
|
||||
@@ -7,14 +7,12 @@ import { z } from 'zod';
|
||||
import {
|
||||
ActionForbidden,
|
||||
CantUpdateOnetimePaymentSubscription,
|
||||
Config,
|
||||
CustomerPortalCreateFailed,
|
||||
InternalServerError,
|
||||
InvalidCheckoutParameters,
|
||||
InvalidLicenseSessionId,
|
||||
InvalidSubscriptionParameters,
|
||||
LicenseRevealed,
|
||||
Mutex,
|
||||
OnEvent,
|
||||
SameSubscriptionRecurring,
|
||||
SubscriptionExpired,
|
||||
@@ -46,9 +44,8 @@ import {
|
||||
SelfhostTeamSubscriptionManager,
|
||||
} from './manager/selfhost';
|
||||
import { ScheduleManager } from './schedule';
|
||||
import { StripeFactory } from './stripe';
|
||||
import {
|
||||
decodeLookupKey,
|
||||
DEFAULT_PRICES,
|
||||
KnownStripeInvoice,
|
||||
KnownStripePrice,
|
||||
KnownStripeSubscription,
|
||||
@@ -57,7 +54,6 @@ import {
|
||||
SubscriptionPlan,
|
||||
SubscriptionRecurring,
|
||||
SubscriptionStatus,
|
||||
SubscriptionVariant,
|
||||
} from './types';
|
||||
|
||||
export const CheckoutExtraArgs = z.union([
|
||||
@@ -75,24 +71,22 @@ export const SubscriptionIdentity = z.union([
|
||||
export { CheckoutParams };
|
||||
|
||||
@Injectable()
|
||||
export class SubscriptionService implements OnApplicationBootstrap {
|
||||
export class SubscriptionService {
|
||||
private readonly logger = new Logger(SubscriptionService.name);
|
||||
private readonly scheduleManager = new ScheduleManager(this.stripe);
|
||||
private readonly scheduleManager = new ScheduleManager(this.stripeProvider);
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly stripe: Stripe,
|
||||
private readonly stripeProvider: StripeFactory,
|
||||
private readonly db: PrismaClient,
|
||||
private readonly feature: FeatureService,
|
||||
private readonly models: Models,
|
||||
private readonly userManager: UserSubscriptionManager,
|
||||
private readonly workspaceManager: WorkspaceSubscriptionManager,
|
||||
private readonly selfhostManager: SelfhostTeamSubscriptionManager,
|
||||
private readonly mutex: Mutex
|
||||
private readonly selfhostManager: SelfhostTeamSubscriptionManager
|
||||
) {}
|
||||
|
||||
async onApplicationBootstrap() {
|
||||
await this.initStripeProducts();
|
||||
get stripe() {
|
||||
return this.stripeProvider.stripe;
|
||||
}
|
||||
|
||||
private select(plan: SubscriptionPlan): SubscriptionManager {
|
||||
@@ -132,8 +126,8 @@ export class SubscriptionService implements OnApplicationBootstrap {
|
||||
const { plan, recurring, variant } = params;
|
||||
|
||||
if (
|
||||
this.config.deploy &&
|
||||
this.config.affine.canary &&
|
||||
env.namespaces.canary &&
|
||||
env.prod &&
|
||||
args.user &&
|
||||
!this.feature.isStaff(args.user.email)
|
||||
) {
|
||||
@@ -683,72 +677,4 @@ export class SubscriptionService implements OnApplicationBootstrap {
|
||||
throw new InvalidSubscriptionParameters();
|
||||
}
|
||||
}
|
||||
|
||||
private async initStripeProducts() {
|
||||
// only init stripe products in dev mode or canary deployment
|
||||
if (
|
||||
(this.config.deploy && !this.config.affine.canary) ||
|
||||
!this.config.node.dev
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
await using lock = await this.mutex.acquire('init stripe prices');
|
||||
|
||||
if (!lock) {
|
||||
return;
|
||||
}
|
||||
|
||||
const keys = new Set<string>();
|
||||
try {
|
||||
await this.stripe.prices
|
||||
.list({
|
||||
active: true,
|
||||
limit: 100,
|
||||
})
|
||||
.autoPagingEach(item => {
|
||||
if (item.lookup_key) {
|
||||
keys.add(item.lookup_key);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
this.logger.warn('Failed to list stripe prices, skip auto init.');
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, setting] of DEFAULT_PRICES) {
|
||||
if (keys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lookupKey = decodeLookupKey(key);
|
||||
|
||||
try {
|
||||
await this.stripe.prices.create({
|
||||
product_data: {
|
||||
name: setting.product,
|
||||
},
|
||||
billing_scheme: 'per_unit',
|
||||
unit_amount: setting.price,
|
||||
currency: 'usd',
|
||||
lookup_key: key,
|
||||
tax_behavior: 'inclusive',
|
||||
recurring:
|
||||
lookupKey.recurring === SubscriptionRecurring.Lifetime ||
|
||||
lookupKey.variant === SubscriptionVariant.Onetime
|
||||
? undefined
|
||||
: {
|
||||
interval:
|
||||
lookupKey.recurring === SubscriptionRecurring.Monthly
|
||||
? 'month'
|
||||
: 'year',
|
||||
interval_count: 1,
|
||||
usage_type: 'licensed',
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
this.logger.error('Failed to create stripe price.', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,130 @@
|
||||
import assert from 'node:assert';
|
||||
|
||||
import { FactoryProvider } from '@nestjs/common';
|
||||
import { omit } from 'lodash-es';
|
||||
import { FactoryProvider, Injectable, Logger } from '@nestjs/common';
|
||||
import Stripe from 'stripe';
|
||||
|
||||
import { Config } from '../../base';
|
||||
import { Config, Mutex, OnEvent } from '../../base';
|
||||
import { ServerFeature, ServerService } from '../../core';
|
||||
import {
|
||||
decodeLookupKey,
|
||||
DEFAULT_PRICES,
|
||||
SubscriptionRecurring,
|
||||
SubscriptionVariant,
|
||||
} from './types';
|
||||
|
||||
@Injectable()
|
||||
export class StripeFactory {
|
||||
#stripe!: Stripe;
|
||||
readonly #logger = new Logger(StripeFactory.name);
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly mutex: Mutex,
|
||||
private readonly server: ServerService
|
||||
) {}
|
||||
|
||||
get stripe() {
|
||||
return this.#stripe;
|
||||
}
|
||||
|
||||
@OnEvent('config.init')
|
||||
async onConfigInit() {
|
||||
this.setup();
|
||||
await this.initStripeProducts();
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
async onConfigChanged(event: Events['config.changed']) {
|
||||
if ('payment' in event.updates) {
|
||||
this.setup();
|
||||
}
|
||||
}
|
||||
|
||||
setup() {
|
||||
// TODO@(@forehalo): use per-requests api key injection
|
||||
this.#stripe = new Stripe(
|
||||
this.config.payment.apiKey ||
|
||||
// NOTE(@forehalo):
|
||||
// we always fake a key if not set because `new Stripe` will complain if it's empty string
|
||||
// this will make code cleaner than providing `Stripe` instance as optional one.
|
||||
'stripe-api-key',
|
||||
this.config.payment.stripe
|
||||
);
|
||||
if (this.config.payment.enabled) {
|
||||
this.server.enableFeature(ServerFeature.Payment);
|
||||
} else {
|
||||
this.server.disableFeature(ServerFeature.Payment);
|
||||
}
|
||||
}
|
||||
|
||||
private async initStripeProducts() {
|
||||
// only init stripe products in dev mode or canary deployment
|
||||
if (!this.config.payment.enabled && !env.namespaces.canary) {
|
||||
return;
|
||||
}
|
||||
|
||||
await using lock = await this.mutex.acquire('init stripe prices');
|
||||
|
||||
if (!lock) {
|
||||
return;
|
||||
}
|
||||
|
||||
const keys = new Set<string>();
|
||||
try {
|
||||
await this.stripe.prices
|
||||
.list({
|
||||
active: true,
|
||||
limit: 100,
|
||||
})
|
||||
.autoPagingEach(item => {
|
||||
if (item.lookup_key) {
|
||||
keys.add(item.lookup_key);
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
this.#logger.warn('Failed to list stripe prices, skip auto init.');
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, setting] of DEFAULT_PRICES) {
|
||||
if (keys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lookupKey = decodeLookupKey(key);
|
||||
|
||||
try {
|
||||
await this.stripe.prices.create({
|
||||
product_data: {
|
||||
name: setting.product,
|
||||
},
|
||||
billing_scheme: 'per_unit',
|
||||
unit_amount: setting.price,
|
||||
currency: 'usd',
|
||||
lookup_key: key,
|
||||
tax_behavior: 'inclusive',
|
||||
recurring:
|
||||
lookupKey.recurring === SubscriptionRecurring.Lifetime ||
|
||||
lookupKey.variant === SubscriptionVariant.Onetime
|
||||
? undefined
|
||||
: {
|
||||
interval:
|
||||
lookupKey.recurring === SubscriptionRecurring.Monthly
|
||||
? 'month'
|
||||
: 'year',
|
||||
interval_count: 1,
|
||||
usage_type: 'licensed',
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
this.#logger.error('Failed to create stripe price.', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const StripeProvider: FactoryProvider = {
|
||||
provide: Stripe,
|
||||
useFactory: (config: Config) => {
|
||||
const stripeConfig = config.plugins.payment.stripe;
|
||||
assert(stripeConfig, 'Stripe configuration is missing');
|
||||
|
||||
return new Stripe(stripeConfig.keys.APIKey, omit(stripeConfig, 'keys'));
|
||||
useFactory: (provider: StripeFactory) => {
|
||||
return provider.stripe;
|
||||
},
|
||||
inject: [Config],
|
||||
inject: [StripeFactory],
|
||||
};
|
||||
|
||||
@@ -3,6 +3,7 @@ import Stripe from 'stripe';
|
||||
|
||||
import { OnEvent } from '../../base';
|
||||
import { SubscriptionService } from './service';
|
||||
import { StripeFactory } from './stripe';
|
||||
|
||||
/**
|
||||
* Stripe webhook events sent in random order, and may be even sent more than once.
|
||||
@@ -14,9 +15,13 @@ import { SubscriptionService } from './service';
|
||||
export class StripeWebhook {
|
||||
constructor(
|
||||
private readonly service: SubscriptionService,
|
||||
private readonly stripe: Stripe
|
||||
private readonly stripeProvider: StripeFactory
|
||||
) {}
|
||||
|
||||
get stripe() {
|
||||
return this.stripeProvider.stripe;
|
||||
}
|
||||
|
||||
@OnEvent('stripe.invoice.created')
|
||||
@OnEvent('stripe.invoice.updated')
|
||||
@OnEvent('stripe.invoice.finalization_failed')
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import { get, merge, omit, set } from 'lodash-es';
|
||||
|
||||
import { OptionalModule, OptionalModuleMetadata } from '../base/nestjs';
|
||||
import { AvailablePlugins } from './config';
|
||||
|
||||
export const REGISTERED_PLUGINS = new Map<AvailablePlugins, AFFiNEModule>();
|
||||
export const ENABLED_PLUGINS = new Set<AvailablePlugins>();
|
||||
|
||||
function registerPlugin(plugin: AvailablePlugins, module: AFFiNEModule) {
|
||||
REGISTERED_PLUGINS.set(plugin, module);
|
||||
}
|
||||
|
||||
interface PluginModuleMetadata extends OptionalModuleMetadata {
|
||||
name: AvailablePlugins;
|
||||
}
|
||||
|
||||
export const Plugin = (options: PluginModuleMetadata) => {
|
||||
return (target: any) => {
|
||||
registerPlugin(options.name, target);
|
||||
|
||||
return OptionalModule(omit(options, 'name'))(target);
|
||||
};
|
||||
};
|
||||
|
||||
export function enablePlugin(plugin: AvailablePlugins, config: any = {}) {
|
||||
config = merge(get(AFFiNE.plugins, plugin), config);
|
||||
set(AFFiNE.plugins, plugin, config);
|
||||
|
||||
ENABLED_PLUGINS.add(plugin);
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import { S3ClientConfig, S3ClientConfigType } from '@aws-sdk/client-s3';
|
||||
|
||||
import { defineStartupConfig, ModuleConfig } from '../../base/config';
|
||||
|
||||
type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__';
|
||||
declare module '../../base/storage/config' {
|
||||
interface StorageProvidersConfig {
|
||||
// the type here is only existing for extends [StorageProviderType] with better type inference and checking.
|
||||
'cloudflare-r2'?: WARNING;
|
||||
'aws-s3'?: WARNING;
|
||||
}
|
||||
}
|
||||
|
||||
export type S3StorageConfig = S3ClientConfigType;
|
||||
export type R2StorageConfig = S3ClientConfigType & {
|
||||
accountId?: string;
|
||||
};
|
||||
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
'aws-s3': ModuleConfig<S3ClientConfig>;
|
||||
'cloudflare-r2': ModuleConfig<R2StorageConfig>;
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.aws-s3', {});
|
||||
defineStartupConfig('plugins.cloudflare-r2', {});
|
||||
@@ -1,42 +0,0 @@
|
||||
import './config';
|
||||
|
||||
import { registerStorageProvider } from '../../base/storage';
|
||||
import { Plugin } from '../registry';
|
||||
import { R2StorageProvider } from './providers/r2';
|
||||
import { S3StorageProvider } from './providers/s3';
|
||||
|
||||
registerStorageProvider('cloudflare-r2', (config, bucket) => {
|
||||
if (!config.plugins['cloudflare-r2']) {
|
||||
throw new Error('Missing cloudflare-r2 storage provider configuration');
|
||||
}
|
||||
|
||||
return new R2StorageProvider(config.plugins['cloudflare-r2'], bucket);
|
||||
});
|
||||
registerStorageProvider('aws-s3', (config, bucket) => {
|
||||
if (!config.plugins['aws-s3']) {
|
||||
throw new Error('Missing aws-s3 storage provider configuration');
|
||||
}
|
||||
|
||||
return new S3StorageProvider(config.plugins['aws-s3'], bucket);
|
||||
});
|
||||
|
||||
@Plugin({
|
||||
name: 'cloudflare-r2',
|
||||
requires: [
|
||||
'plugins.cloudflare-r2.accountId',
|
||||
'plugins.cloudflare-r2.credentials.accessKeyId',
|
||||
'plugins.cloudflare-r2.credentials.secretAccessKey',
|
||||
],
|
||||
if: config => config.flavor.graphql,
|
||||
})
|
||||
export class CloudflareR2Module {}
|
||||
|
||||
@Plugin({
|
||||
name: 'aws-s3',
|
||||
requires: [
|
||||
'plugins.aws-s3.credentials.accessKeyId',
|
||||
'plugins.aws-s3.credentials.secretAccessKey',
|
||||
],
|
||||
if: config => config.flavor.graphql,
|
||||
})
|
||||
export class AwsS3Module {}
|
||||
@@ -1,23 +0,0 @@
|
||||
import assert from 'node:assert';
|
||||
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import type { R2StorageConfig } from '../config';
|
||||
import { S3StorageProvider } from './s3';
|
||||
|
||||
export class R2StorageProvider extends S3StorageProvider {
|
||||
override readonly type = 'cloudflare-r2' as any /* cast 'r2' to 's3' */;
|
||||
|
||||
constructor(config: R2StorageConfig, bucket: string) {
|
||||
assert(config.accountId, 'accountId is required for R2 storage provider');
|
||||
super(
|
||||
{
|
||||
...config,
|
||||
forcePathStyle: true,
|
||||
endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
|
||||
},
|
||||
bucket
|
||||
);
|
||||
this.logger = new Logger(`${R2StorageProvider.name}:${bucket}`);
|
||||
}
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
/* oxlint-disable @typescript-eslint/no-non-null-assertion */
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import {
|
||||
DeleteObjectCommand,
|
||||
GetObjectCommand,
|
||||
HeadObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
NoSuchKey,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import {
|
||||
autoMetadata,
|
||||
BlobInputType,
|
||||
GetObjectMetadata,
|
||||
ListObjectsMetadata,
|
||||
PutObjectMetadata,
|
||||
StorageProvider,
|
||||
toBuffer,
|
||||
} from '../../../base/storage';
|
||||
import type { S3StorageConfig } from '../config';
|
||||
|
||||
export class S3StorageProvider implements StorageProvider {
|
||||
protected logger: Logger;
|
||||
protected client: S3Client;
|
||||
|
||||
readonly type = 'aws-s3';
|
||||
|
||||
constructor(
|
||||
config: S3StorageConfig,
|
||||
public readonly bucket: string
|
||||
) {
|
||||
this.client = new S3Client({
|
||||
region: 'auto',
|
||||
// s3 client uses keep-alive by default to accelerate requests, and max requests queue is 50.
|
||||
// If some of them are long holding or dead without response, the whole queue will block.
|
||||
// By default no timeout is set for requests or connections, so we set them here.
|
||||
requestHandler: { requestTimeout: 60_000, connectionTimeout: 10_000 },
|
||||
...config,
|
||||
});
|
||||
this.logger = new Logger(`${S3StorageProvider.name}:${bucket}`);
|
||||
}
|
||||
|
||||
async put(
|
||||
key: string,
|
||||
body: BlobInputType,
|
||||
metadata: PutObjectMetadata = {}
|
||||
): Promise<void> {
|
||||
const blob = await toBuffer(body);
|
||||
|
||||
metadata = autoMetadata(blob, metadata);
|
||||
|
||||
try {
|
||||
await this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
Body: blob,
|
||||
|
||||
// metadata
|
||||
ContentType: metadata.contentType,
|
||||
ContentLength: metadata.contentLength,
|
||||
// TODO(@forehalo): Cloudflare doesn't support CRC32, use md5 instead later.
|
||||
// ChecksumCRC32: metadata.checksumCRC32,
|
||||
})
|
||||
);
|
||||
|
||||
this.logger.verbose(`Object \`${key}\` put`);
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
`Failed to put object (${JSON.stringify({
|
||||
key,
|
||||
bucket: this.bucket,
|
||||
metadata,
|
||||
})})`
|
||||
);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async head(key: string) {
|
||||
try {
|
||||
const obj = await this.client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
contentType: obj.ContentType!,
|
||||
contentLength: obj.ContentLength!,
|
||||
lastModified: obj.LastModified!,
|
||||
checksumCRC32: obj.ChecksumCRC32,
|
||||
};
|
||||
} catch (e) {
|
||||
// 404
|
||||
if (e instanceof NoSuchKey) {
|
||||
this.logger.verbose(`Object \`${key}\` not found`);
|
||||
return undefined;
|
||||
}
|
||||
this.logger.error(`Failed to head object \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async get(key: string): Promise<{
|
||||
body?: Readable;
|
||||
metadata?: GetObjectMetadata;
|
||||
}> {
|
||||
try {
|
||||
const obj = await this.client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
if (!obj.Body) {
|
||||
this.logger.verbose(`Object \`${key}\` not found`);
|
||||
return {};
|
||||
}
|
||||
|
||||
this.logger.verbose(`Read object \`${key}\``);
|
||||
return {
|
||||
// @ts-expect-errors ignore browser response type `Blob`
|
||||
body: obj.Body,
|
||||
metadata: {
|
||||
// always set when putting object
|
||||
contentType: obj.ContentType!,
|
||||
contentLength: obj.ContentLength!,
|
||||
lastModified: obj.LastModified!,
|
||||
checksumCRC32: obj.ChecksumCRC32,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
// 404
|
||||
if (e instanceof NoSuchKey) {
|
||||
this.logger.verbose(`Object \`${key}\` not found`);
|
||||
return {};
|
||||
}
|
||||
this.logger.error(`Failed to read object \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async list(prefix?: string): Promise<ListObjectsMetadata[]> {
|
||||
// continuationToken should be `string | undefined`,
|
||||
// but TypeScript will fail on type infer in the code below.
|
||||
// Seems to be a bug in TypeScript
|
||||
let continuationToken: any = undefined;
|
||||
let hasMore = true;
|
||||
let result: ListObjectsMetadata[] = [];
|
||||
|
||||
try {
|
||||
while (hasMore) {
|
||||
const listResult = await this.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: this.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
);
|
||||
|
||||
if (listResult.Contents?.length) {
|
||||
result = result.concat(
|
||||
listResult.Contents.map(r => ({
|
||||
key: r.Key!,
|
||||
lastModified: r.LastModified!,
|
||||
contentLength: r.Size!,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// has more items not listed
|
||||
hasMore = !!listResult.IsTruncated;
|
||||
continuationToken = listResult.NextContinuationToken;
|
||||
}
|
||||
|
||||
this.logger.verbose(
|
||||
`List ${result.length} objects with prefix \`${prefix}\``
|
||||
);
|
||||
return result;
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to list objects with prefix \`${prefix}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
try {
|
||||
await this.client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: key,
|
||||
})
|
||||
);
|
||||
|
||||
this.logger.verbose(`Deleted object \`${key}\``);
|
||||
} catch (e) {
|
||||
this.logger.error(`Failed to delete object \`${key}\``);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,20 @@
|
||||
import { defineStartupConfig, ModuleConfig } from '../../base/config';
|
||||
import { defineModuleConfig } from '../../base';
|
||||
|
||||
export interface WorkerStartupConfigurations {
|
||||
allowedOrigin: string[];
|
||||
}
|
||||
|
||||
declare module '../config' {
|
||||
interface PluginsConfig {
|
||||
worker: ModuleConfig<WorkerStartupConfigurations>;
|
||||
declare global {
|
||||
interface AppConfigSchema {
|
||||
worker: {
|
||||
allowedOrigin: ConfigItem<string[]>;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
defineStartupConfig('plugins.worker', {
|
||||
allowedOrigin: ['localhost', '127.0.0.1'],
|
||||
defineModuleConfig('worker', {
|
||||
allowedOrigin: {
|
||||
desc: 'Allowed origin',
|
||||
default: ['localhost', '127.0.0.1'],
|
||||
},
|
||||
});
|
||||
|
||||
@@ -10,8 +10,9 @@ import {
|
||||
import type { Request, Response } from 'express';
|
||||
import { HTMLRewriter } from 'htmlrewriter';
|
||||
|
||||
import { BadRequest, Cache, Config, URLHelper } from '../../base';
|
||||
import { BadRequest, Cache, URLHelper, UseNamedGuard } from '../../base';
|
||||
import { Public } from '../../core/auth';
|
||||
import { WorkerService } from './service';
|
||||
import type { LinkPreviewRequest, LinkPreviewResponse } from './types';
|
||||
import {
|
||||
appendUrl,
|
||||
@@ -20,7 +21,6 @@ import {
|
||||
getCorsHeaders,
|
||||
isOriginAllowed,
|
||||
isRefererAllowed,
|
||||
OriginRules,
|
||||
parseJson,
|
||||
reduceUrls,
|
||||
} from './utils';
|
||||
@@ -30,22 +30,19 @@ import { decodeWithCharset } from './utils/encoding';
|
||||
const CACHE_TTL = 1000 * 60 * 30;
|
||||
|
||||
@Public()
|
||||
@UseNamedGuard('selfhost')
|
||||
@Controller('/api/worker')
|
||||
export class WorkerController {
|
||||
private readonly logger = new Logger(WorkerController.name);
|
||||
private readonly allowedOrigin: OriginRules;
|
||||
|
||||
constructor(
|
||||
config: Config,
|
||||
private readonly cache: Cache,
|
||||
private readonly url: URLHelper
|
||||
) {
|
||||
this.allowedOrigin = [
|
||||
...config.plugins.worker.allowedOrigin
|
||||
.map(u => fixUrl(u)?.origin as string)
|
||||
.filter(v => !!v),
|
||||
url.origin,
|
||||
];
|
||||
private readonly url: URLHelper,
|
||||
private readonly service: WorkerService
|
||||
) {}
|
||||
|
||||
private get allowedOrigin() {
|
||||
return this.service.allowedOrigins;
|
||||
}
|
||||
|
||||
@Get('/image-proxy')
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import './config';
|
||||
|
||||
import { Plugin } from '../registry';
|
||||
import { WorkerController } from './controller';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
@Plugin({
|
||||
name: 'worker',
|
||||
import { WorkerController } from './controller';
|
||||
import { WorkerService } from './service';
|
||||
@Module({
|
||||
providers: [WorkerService],
|
||||
controllers: [WorkerController],
|
||||
if: config => config.isSelfhosted || config.node.dev || config.node.test,
|
||||
})
|
||||
export class WorkerModule {}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Config, OnEvent, URLHelper } from '../../base';
|
||||
import { fixUrl, OriginRules } from './utils';
|
||||
|
||||
@Injectable()
|
||||
export class WorkerService {
|
||||
allowedOrigins: OriginRules = [this.url.origin];
|
||||
|
||||
constructor(
|
||||
private readonly config: Config,
|
||||
private readonly url: URLHelper
|
||||
) {}
|
||||
|
||||
@OnEvent('config.init')
|
||||
onConfigInit() {
|
||||
this.allowedOrigins = [
|
||||
...this.config.worker.allowedOrigin
|
||||
.map(u => fixUrl(u)?.origin as string)
|
||||
.filter(v => !!v),
|
||||
this.url.origin,
|
||||
];
|
||||
}
|
||||
|
||||
@OnEvent('config.changed')
|
||||
onConfigChanged(event: Events['config.changed']) {
|
||||
if ('worker' in event.updates) {
|
||||
this.onConfigInit();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user