feat(server): runtime setting support (#5602)

---

<details open="true"><summary>Generated summary (powered by <a href="https://app.graphite.dev">Graphite</a>)</summary>

> ## TL;DR
> This pull request adds a new migration file, a new model, and new modules related to runtime settings. It also introduces a new `Runtime` service that allows getting, setting, and updating runtime configurations.
>
> ## What changed
> - Added a new migration file `migration.sql` that creates a table called `application_settings` with columns `key` and `value`.
> - Added a new model `ApplicationSetting` with properties `key` and `value`.
> - Added a new module `RuntimeSettingModule` that exports the `Runtime` service.
> - Added a new service `Runtime` that provides methods for getting, setting, and updating runtime configurations.
> - Modified the `app.module.ts` file to import the `RuntimeSettingModule`.
> - Modified the `index.ts` file in the `fundamentals` directory to export the `Runtime` service.
> - Added a new file `def.ts` in the `runtime` directory that defines the runtime configurations and provides a default implementation.
> - Added a new file `service.ts` in the `runtime` directory that implements the `Runtime` service.
>
> ## How to test
> 1. Run the migration script to create the `application_settings` table.
> 2. Use the `Runtime` service to get, set, and update runtime configurations.
> 3. Verify that the runtime configurations are stored correctly in the database and can be retrieved and modified using the `Runtime` service.
>
> ## Why make this change
> This change introduces a new feature related to runtime settings. The `Runtime` service allows the application to dynamically manage and modify runtime configurations without requiring a restart. This provides flexibility and allows for easier customization and configuration of the application.
</details>
This commit is contained in:
forehalo
2024-05-28 06:43:53 +00:00
parent 9d296c4b62
commit 638fc62601
116 changed files with 1907 additions and 1106 deletions

View File

@@ -1,30 +1,20 @@
import { CopilotConfig } from './copilot';
import { GCloudConfig } from './gcloud/config';
import { OAuthConfig } from './oauth';
import { PaymentConfig } from './payment';
import { RedisOptions } from './redis';
import { R2StorageConfig, S3StorageConfig } from './storage';
import { ModuleStartupConfigDescriptions } from '../fundamentals/config/types';
export interface PluginsConfig {}
export type AvailablePlugins = keyof PluginsConfig;
declare module '../fundamentals/config' {}
declare module '../fundamentals/config' {
interface PluginsConfig {
readonly copilot: CopilotConfig;
readonly payment: PaymentConfig;
readonly redis: RedisOptions;
readonly gcloud: GCloudConfig;
readonly 'cloudflare-r2': R2StorageConfig;
readonly 'aws-s3': S3StorageConfig;
readonly oauth: OAuthConfig;
interface AppConfig {
plugins: PluginsConfig;
}
export type AvailablePlugins = keyof PluginsConfig;
interface AFFiNEConfig {
readonly plugins: {
enabled: Set<AvailablePlugins>;
use<Plugin extends AvailablePlugins>(
plugin: Plugin,
config?: DeepPartial<PluginsConfig[Plugin]>
): void;
} & Partial<PluginsConfig>;
interface AppPluginsConfig {
use<Plugin extends AvailablePlugins>(
plugin: Plugin,
config?: DeepPartial<
ModuleStartupConfigDescriptions<PluginsConfig[Plugin]>
>
): void;
}
}

View File

@@ -0,0 +1,26 @@
import type { ClientOptions as OpenAIClientOptions } from 'openai';
import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config';
import { StorageConfig } from '../../fundamentals/storage/config';
import type { FalConfig } from './providers/fal';
export interface CopilotStartupConfigurations {
openai?: OpenAIClientOptions;
fal?: FalConfig;
test?: never;
unsplashKey?: string;
storage: StorageConfig;
}
declare module '../config' {
interface PluginsConfig {
copilot: ModuleConfig<CopilotStartupConfigurations>;
}
}
defineStartupConfig('plugins.copilot', {
storage: {
provider: 'fs',
bucket: 'copilot',
},
});

View File

@@ -1,3 +1,5 @@
import './config';
import { ServerFeature } from '../../core/config';
import { FeatureModule } from '../../core/features';
import { QuotaModule } from '../../core/quota';
@@ -43,5 +45,3 @@ registerCopilotProvider(OpenAIProvider);
},
})
export class CopilotModule {}
export type { CopilotConfig } from './types';

View File

@@ -2,16 +2,17 @@ import assert from 'node:assert';
import { Injectable, Logger } from '@nestjs/common';
import { Config } from '../../../fundamentals';
import { AFFiNEConfig, Config } from '../../../fundamentals';
import { CopilotStartupConfigurations } from '../config';
import {
CapabilityToCopilotProvider,
CopilotCapability,
CopilotConfig,
CopilotProvider,
CopilotProviderType,
} from '../types';
type CopilotProviderConfig = CopilotConfig[keyof CopilotConfig];
type CopilotProviderConfig =
CopilotStartupConfigurations[keyof CopilotStartupConfigurations];
interface CopilotProviderDefinition<C extends CopilotProviderConfig> {
// constructor signature
@@ -37,7 +38,10 @@ const PROVIDER_CAPABILITY_MAP = new Map<
>();
// config assertions for providers
const ASSERT_CONFIG = new Map<CopilotProviderType, (config: Config) => void>();
const ASSERT_CONFIG = new Map<
CopilotProviderType,
(config: AFFiNEConfig) => void
>();
export function registerCopilotProvider<
C extends CopilotProviderConfig = CopilotProviderConfig,
@@ -69,7 +73,7 @@ export function registerCopilotProvider<
PROVIDER_CAPABILITY_MAP.set(capability, providers);
}
// register the provider config assertion
ASSERT_CONFIG.set(type, (config: Config) => {
ASSERT_CONFIG.set(type, (config: AFFiNEConfig) => {
assert(config.plugins.copilot);
const providerConfig = config.plugins.copilot[type];
if (!providerConfig) return false;
@@ -89,7 +93,7 @@ export function unregisterCopilotProvider(type: CopilotProviderType) {
}
/// Asserts that the config is valid for any registered providers
export function assertProvidersConfigs(config: Config) {
export function assertProvidersConfigs(config: AFFiNEConfig) {
return (
Array.from(ASSERT_CONFIG.values()).findIndex(assertConfig =>
assertConfig(config)

View File

@@ -9,6 +9,7 @@ import {
type FileUpload,
type StorageProvider,
StorageProviderFactory,
URLHelper,
} from '../../fundamentals';
@Injectable()
@@ -17,10 +18,13 @@ export class CopilotStorage {
constructor(
private readonly config: Config,
private readonly url: URLHelper,
private readonly storageFactory: StorageProviderFactory,
private readonly quota: QuotaManagementService
) {
this.provider = this.storageFactory.create('copilot');
this.provider = this.storageFactory.create(
this.config.plugins.copilot.storage
);
}
async put(
@@ -35,7 +39,7 @@ export class CopilotStorage {
// return image base64url for dev environment
return `data:image/png;base64,${blob.toString('base64')}`;
}
return `${this.config.baseUrl}/api/copilot/blob/${name}`;
return this.url.link(`/api/copilot/blob/${name}`);
}
async get(userId: string, workspaceId: string, key: string) {

View File

@@ -1,18 +1,9 @@
import { type Tokenizer } from '@affine/server-native';
import { AiPromptRole } from '@prisma/client';
import type { ClientOptions as OpenAIClientOptions } from 'openai';
import { z } from 'zod';
import { fromModelName } from '../../native';
import type { ChatPrompt } from './prompt';
import type { FalConfig } from './providers/fal';
export interface CopilotConfig {
openai: OpenAIClientOptions;
fal: FalConfig;
unsplashKey: string;
test: never;
}
export enum AvailableModels {
// text to text

View File

@@ -1 +1,14 @@
export interface GCloudConfig {}
import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config';
export interface GCloudConfig {
enabled: boolean;
}
declare module '../config' {
interface PluginsConfig {
gcloud: ModuleConfig<GCloudConfig>;
}
}
defineStartupConfig('plugins.gcloud', {
enabled: false,
});

View File

@@ -1,3 +1,5 @@
import './config';
import { Global } from '@nestjs/common';
import { Plugin } from '../registry';

View File

@@ -5,4 +5,8 @@ import './payment';
import './redis';
import './storage';
export { REGISTERED_PLUGINS } from './registry';
export {
enablePlugin,
REGISTERED_PLUGINS,
ENABLED_PLUGINS as USED_PLUGINS,
} from './registry';

View File

@@ -1,3 +1,5 @@
import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config';
export interface OAuthProviderConfig {
clientId: string;
clientSecret: string;
@@ -29,6 +31,15 @@ type OAuthProviderConfigMapping = {
};
export interface OAuthConfig {
enabled: boolean;
providers: Partial<OAuthProviderConfigMapping>;
}
declare module '../config' {
interface PluginsConfig {
oauth: ModuleConfig<OAuthConfig>;
}
}
defineStartupConfig('plugins.oauth', {
providers: {},
});

View File

@@ -12,10 +12,10 @@ import type { Request, Response } from 'express';
import { AuthService, Public } from '../../core/auth';
import { UserService } from '../../core/user';
import { URLHelper } from '../../fundamentals';
import { OAuthProviderName } from './config';
import { OAuthAccount, Tokens } from './providers/def';
import { OAuthProviderFactory } from './register';
import { OAuthService } from './service';
import { OAuthProviderName } from './types';
@Controller('/oauth')
export class OAuthController {

View File

@@ -1,3 +1,5 @@
import './config';
import { AuthModule } from '../../core/auth';
import { ServerFeature } from '../../core/config';
import { UserModule } from '../../core/user';
@@ -22,4 +24,3 @@ import { OAuthService } from './service';
if: config => !!config.plugins.oauth,
})
export class OAuthModule {}
export type { OAuthConfig } from './types';

View File

@@ -1,4 +1,4 @@
import { OAuthProviderName } from '../types';
import { OAuthProviderName } from '../config';
export interface OAuthAccount {
id: string;

View File

@@ -1,8 +1,8 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { Config, URLHelper } from '../../../fundamentals';
import { OAuthProviderName } from '../config';
import { AutoRegisteredOAuthProvider } from '../register';
import { OAuthProviderName } from '../types';
interface AuthTokenResponse {
access_token: string;

View File

@@ -1,8 +1,8 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { Config, URLHelper } from '../../../fundamentals';
import { OAuthProviderName } from '../config';
import { AutoRegisteredOAuthProvider } from '../register';
import { OAuthProviderName } from '../types';
interface GoogleOAuthTokenResponse {
access_token: string;

View File

@@ -7,8 +7,12 @@ import {
import { z } from 'zod';
import { Config, URLHelper } from '../../../fundamentals';
import {
OAuthOIDCProviderConfig,
OAuthProviderName,
OIDCArgs,
} from '../config';
import { AutoRegisteredOAuthProvider } from '../register';
import { OAuthOIDCProviderConfig, OAuthProviderName, OIDCArgs } from '../types';
import { OAuthAccount, Tokens } from './def';
const OIDCTokenSchema = z.object({

View File

@@ -1,8 +1,8 @@
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { Config } from '../../fundamentals';
import { OAuthProviderName } from './config';
import { OAuthProvider } from './providers/def';
import { OAuthProviderName } from './types';
const PROVIDERS: Map<OAuthProviderName, OAuthProvider> = new Map();

View File

@@ -1,8 +1,8 @@
import { registerEnumType, ResolveField, Resolver } from '@nestjs/graphql';
import { ServerConfigType } from '../../core/config';
import { OAuthProviderName } from './config';
import { OAuthProviderFactory } from './register';
import { OAuthProviderName } from './types';
registerEnumType(OAuthProviderName, { name: 'OAuthProviderType' });

View File

@@ -3,8 +3,8 @@ import { randomUUID } from 'node:crypto';
import { Injectable } from '@nestjs/common';
import { SessionCache } from '../../fundamentals';
import { OAuthProviderName } from './config';
import { OAuthProviderFactory } from './register';
import { OAuthProviderName } from './types';
const OAUTH_STATE_KEY = 'OAUTH_STATE';

View File

@@ -0,0 +1,20 @@
import type { Stripe } from 'stripe';
import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config';
export interface PaymentStartupConfig {
stripe?: {
keys: {
APIKey: string;
webhookKey: string;
};
} & Stripe.StripeConfig;
}
declare module '../config' {
interface PluginsConfig {
payment: ModuleConfig<PaymentStartupConfig>;
}
}
defineStartupConfig('plugins.payment', {});

View File

@@ -1,3 +1,5 @@
import './config';
import { ServerFeature } from '../../core/config';
import { FeatureModule } from '../../core/features';
import { Plugin } from '../registry';
@@ -26,5 +28,3 @@ import { StripeWebhook } from './webhook';
if: config => config.flavor.graphql,
})
export class PaymentModule {}
export type { PaymentConfig } from './types';

View File

@@ -19,7 +19,7 @@ import { groupBy } from 'lodash-es';
import { CurrentUser, Public } from '../../core/auth';
import { UserType } from '../../core/user';
import { Config } from '../../fundamentals';
import { Config, URLHelper } from '../../fundamentals';
import { decodeLookupKey, SubscriptionService } from './service';
import {
InvoiceStatus,
@@ -146,8 +146,8 @@ class CreateCheckoutSessionInput {
@Field(() => String, { nullable: true })
coupon!: string | null;
@Field(() => String, { nullable: true })
successCallbackLink!: string | null;
@Field(() => String)
successCallbackLink!: string;
// @FIXME(forehalo): we should put this field in the header instead of as a explicity args
@Field(() => String)
@@ -158,7 +158,7 @@ class CreateCheckoutSessionInput {
export class SubscriptionResolver {
constructor(
private readonly service: SubscriptionService,
private readonly config: Config
private readonly url: URLHelper
) {}
@Public()
@@ -222,8 +222,7 @@ export class SubscriptionResolver {
plan: input.plan,
recurring: input.recurring,
promotionCode: input.coupon,
redirectUrl:
input.successCallbackLink ?? `${this.config.baseUrl}/upgrade-success`,
redirectUrl: this.url.link(input.successCallbackLink),
idempotencyKey: input.idempotencyKey,
});

View File

@@ -9,8 +9,8 @@ import { Config } from '../../fundamentals';
export const StripeProvider: FactoryProvider = {
provide: Stripe,
useFactory: (config: Config) => {
assert(config.plugins.payment);
const stripeConfig = config.plugins.payment.stripe;
assert(stripeConfig, 'Stripe configuration is missing');
return new Stripe(stripeConfig.keys.APIKey, omit(stripeConfig, 'keys'));
},

View File

@@ -1,17 +1,7 @@
import type { User } from '@prisma/client';
import type { Stripe } from 'stripe';
import type { Payload } from '../../fundamentals/event/def';
export interface PaymentConfig {
stripe: {
keys: {
APIKey: string;
webhookKey: string;
};
} & Stripe.StripeConfig;
}
export enum SubscriptionRecurring {
Monthly = 'monthly',
Yearly = 'yearly',

View File

@@ -25,7 +25,7 @@ export class StripeWebhook {
private readonly stripe: Stripe,
private readonly event: EventEmitter2
) {
assert(config.plugins.payment);
assert(config.plugins.payment.stripe);
this.webhookKey = config.plugins.payment.stripe.keys.webhookKey;
}

View File

@@ -0,0 +1,11 @@
import { RedisOptions } from 'ioredis';
import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config';
declare module '../config' {
interface PluginsConfig {
redis: ModuleConfig<RedisOptions>;
}
}
defineStartupConfig('plugins.redis', {});

View File

@@ -1,5 +1,6 @@
import './config';
import { Global, Provider, Type } from '@nestjs/common';
import type { RedisOptions } from 'ioredis';
import { Redis } from 'ioredis';
import { ThrottlerStorageRedisService } from 'nestjs-throttler-storage-redis';
@@ -64,5 +65,3 @@ const mutexRedisAdapterProvider: Provider = {
requires: ['plugins.redis.host'],
})
export class RedisModule {}
export { RedisOptions };

View File

@@ -30,20 +30,20 @@ class Redis extends IORedis implements OnModuleDestroy, OnModuleInit {
@Injectable()
export class CacheRedis extends Redis {
constructor(config: Config) {
super(config.plugins.redis ?? {});
super(config.plugins.redis);
}
}
@Injectable()
export class SessionRedis extends Redis {
constructor(config: Config) {
super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 2 });
super({ ...config.plugins.redis, db: (config.plugins.redis.db ?? 0) + 2 });
}
}
@Injectable()
export class SocketIoRedis extends Redis {
constructor(config: Config) {
super({ ...config.plugins.redis, db: (config.plugins.redis?.db ?? 0) + 3 });
super({ ...config.plugins.redis, db: (config.plugins.redis.db ?? 0) + 3 });
}
}

View File

@@ -1,3 +0,0 @@
import { RedisOptions } from 'ioredis';
export type { RedisOptions };

View File

@@ -1,11 +1,12 @@
import { omit } from 'lodash-es';
import { get, merge, omit, set } from 'lodash-es';
import { AvailablePlugins } from '../fundamentals/config';
import { OptionalModule, OptionalModuleMetadata } from '../fundamentals/nestjs';
import { AvailablePlugins } from './config';
export const REGISTERED_PLUGINS = new Map<AvailablePlugins, AFFiNEModule>();
export const ENABLED_PLUGINS = new Set<AvailablePlugins>();
function register(plugin: AvailablePlugins, module: AFFiNEModule) {
function registerPlugin(plugin: AvailablePlugins, module: AFFiNEModule) {
REGISTERED_PLUGINS.set(plugin, module);
}
@@ -15,8 +16,15 @@ interface PluginModuleMetadata extends OptionalModuleMetadata {
export const Plugin = (options: PluginModuleMetadata) => {
return (target: any) => {
register(options.name, target);
registerPlugin(options.name, target);
return OptionalModule(omit(options, 'name'))(target);
};
};
export function enablePlugin(plugin: AvailablePlugins, config: any = {}) {
config = merge(get(AFFiNE.plugins, plugin), config);
set(AFFiNE.plugins, plugin, config);
ENABLED_PLUGINS.add(plugin);
}

View File

@@ -0,0 +1,27 @@
import { S3ClientConfig, S3ClientConfigType } from '@aws-sdk/client-s3';
import { defineStartupConfig, ModuleConfig } from '../../fundamentals/config';
type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__';
declare module '../../fundamentals/storage/config' {
interface StorageProvidersConfig {
// the type here is only existing for extends [StorageProviderType] with better type inference and checking.
'cloudflare-r2'?: WARNING;
'aws-s3'?: WARNING;
}
}
export type S3StorageConfig = S3ClientConfigType;
export type R2StorageConfig = S3ClientConfigType & {
accountId?: string;
};
declare module '../config' {
interface PluginsConfig {
'aws-s3': ModuleConfig<S3ClientConfig>;
'cloudflare-r2': ModuleConfig<R2StorageConfig>;
}
}
defineStartupConfig('plugins.aws-s3', {});
defineStartupConfig('plugins.cloudflare-r2', {});

View File

@@ -1,3 +1,5 @@
import './config';
import { registerStorageProvider } from '../../fundamentals/storage';
import { Plugin } from '../registry';
import { R2StorageProvider } from './providers/r2';
@@ -38,5 +40,3 @@ export class CloudflareR2Module {}
if: config => config.flavor.graphql,
})
export class AwsS3Module {}
export type { R2StorageConfig, S3StorageConfig } from './types';

View File

@@ -1,12 +1,15 @@
import assert from 'node:assert';
import { Logger } from '@nestjs/common';
import type { R2StorageConfig } from '../types';
import type { R2StorageConfig } from '../config';
import { S3StorageProvider } from './s3';
export class R2StorageProvider extends S3StorageProvider {
override readonly type = 'cloudflare-r2' as any /* cast 'r2' to 's3' */;
constructor(config: R2StorageConfig, bucket: string) {
assert(config.accountId, 'accountId is required for R2 storage provider');
super(
{
...config,

View File

@@ -20,7 +20,7 @@ import {
StorageProvider,
toBuffer,
} from '../../../fundamentals/storage';
import type { S3StorageConfig } from '../types';
import type { S3StorageConfig } from '../config';
export class S3StorageProvider implements StorageProvider {
protected logger: Logger;

View File

@@ -1,16 +0,0 @@
import { S3ClientConfigType } from '@aws-sdk/client-s3';
type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__';
export type R2StorageConfig = S3ClientConfigType & {
accountId: string;
};
export type S3StorageConfig = S3ClientConfigType;
declare module '../../fundamentals/config/storage' {
interface StorageProvidersConfig {
// the type here is only existing for extends [StorageProviderType] with better type inference and checking.
'cloudflare-r2'?: WARNING;
'aws-s3'?: WARNING;
}
}