From 0ea38680fa2aecb1545b451f1c511283ae4b776e Mon Sep 17 00:00:00 2001 From: forehalo Date: Thu, 27 Mar 2025 12:32:28 +0000 Subject: [PATCH] refactor(server): config system (#11081) --- .docker/selfhost/schema.json | 900 +++++++++ .github/actions/deploy/deploy.mjs | 57 +- .../charts/doc/templates/deployment.yaml | 48 +- .../templates/deployment.yaml | 6 +- .../charts/gcloud-sql-proxy/values.yaml | 1 + .../graphql/templates/captcha-secret.yaml | 9 - .../graphql/templates/copilot-secret.yaml | 13 - .../charts/graphql/templates/deployment.yaml | 133 +- .../charts/graphql/templates/mailer.yaml | 13 - .../graphql/templates/metrics-secret.yaml | 9 - .../charts/graphql/templates/migration.yaml | 34 +- .../charts/graphql/templates/oauth.yaml | 21 - .../charts/graphql/templates/payment.yml | 8 - .../helm/affine/charts/graphql/values.yaml | 45 +- .../charts/renderer/templates/deployment.yaml | 23 +- .../charts/sync/templates/deployment.yaml | 6 +- .github/helm/affine/templates/r2-secret.yaml | 11 - .github/helm/affine/values.yaml | 23 +- .prettierignore | 1 + oxlint.json | 3 +- .../migration.sql | 13 + packages/backend/server/package.json | 6 +- packages/backend/server/schema.gql | 1767 +++++++++++++++++ packages/backend/server/schema.prisma | 67 +- packages/backend/server/scripts/genconfig.ts | 101 + .../server/scripts/self-host-predeploy.js | 18 +- .../server/src/__tests__/app/doc.e2e.ts | 36 - .../server/src/__tests__/app/graphql.e2e.ts | 137 -- .../server/src/__tests__/app/renderer.e2e.ts | 36 - .../server/src/__tests__/app/selfhost.e2e.ts | 9 +- .../server/src/__tests__/app/sync.e2e.ts | 36 - .../src/__tests__/auth/controller.spec.ts | 7 +- .../server/src/__tests__/config.spec.ts | 39 - .../src/__tests__/copilot-provider.spec.ts | 40 +- .../server/src/__tests__/copilot.e2e.ts | 51 +- .../server/src/__tests__/copilot.spec.ts | 151 +- .../server/src/__tests__/create-module.ts | 18 +- .../server/src/__tests__/doc/renderer.spec.ts | 7 +- .../src/__tests__/e2e/{ => apps}/app.spec.ts | 11 +- .../src/__tests__/e2e/apps/flavors.spec.ts | 46 + .../server/src/__tests__/e2e/create-app.ts | 11 +- .../backend/server/src/__tests__/env.spec.ts | 156 ++ .../src/__tests__/mocks/copilot.mock.ts | 113 ++ .../src/__tests__/mocks/eventbus.mock.ts | 35 + .../server/src/__tests__/mocks/index.ts | 4 +- .../src/__tests__/models/feature-user.spec.ts | 11 +- .../src/__tests__/nestjs/throttler.spec.ts | 14 +- .../src/__tests__/oauth/controller.spec.ts | 16 +- .../src/__tests__/payment/service.spec.ts | 72 +- .../server/src/__tests__/utils/common.ts | 1 - .../server/src/__tests__/utils/copilot.ts | 153 +- .../server/src/__tests__/utils/testing-app.ts | 9 +- .../src/__tests__/utils/testing-module.ts | 17 +- .../server/src/__tests__/utils/utils.ts | 19 + .../server/src/__tests__/version.spec.ts | 72 +- .../server/src/__tests__/worker.e2e.ts | 7 +- packages/backend/server/src/app.controller.ts | 12 +- packages/backend/server/src/app.module.ts | 160 +- packages/backend/server/src/app.ts | 19 +- .../src/base/config/__tests__/config.spec.ts | 90 + .../backend/server/src/base/config/config.ts | 3 + .../backend/server/src/base/config/def.ts | 55 - .../backend/server/src/base/config/default.ts | 142 -- .../backend/server/src/base/config/env.ts | 43 +- .../backend/server/src/base/config/factory.ts | 60 + .../backend/server/src/base/config/index.ts | 46 +- .../server/src/base/config/provider.ts | 26 +- .../server/src/base/config/register.ts | 279 ++- .../backend/server/src/base/config/types.ts | 135 +- packages/backend/server/src/base/error/def.ts | 6 + .../server/src/base/error/errors.gen.ts | 9 +- .../backend/server/src/base/error/index.ts | 4 +- packages/backend/server/src/base/event/def.ts | 2 +- .../backend/server/src/base/event/index.ts | 5 +- .../backend/server/src/base/event/scanner.ts | 2 +- .../backend/server/src/base/graphql/config.ts | 28 +- .../backend/server/src/base/graphql/index.ts | 22 +- .../server/src/base/graphql/register.ts | 3 +- .../src/base/helpers/__tests__/crypto.spec.ts | 39 +- .../backend/server/src/base/helpers/config.ts | 61 +- .../backend/server/src/base/helpers/crypto.ts | 68 +- .../backend/server/src/base/helpers/url.ts | 15 +- packages/backend/server/src/base/index.ts | 17 +- .../base/job/queue/__tests__/queue.spec.ts | 16 +- .../server/src/base/job/queue/config.ts | 113 +- .../backend/server/src/base/job/queue/def.ts | 2 +- .../server/src/base/job/queue/executor.ts | 91 +- .../backend/server/src/base/metrics/config.ts | 35 +- .../backend/server/src/base/metrics/index.ts | 48 +- .../server/src/base/metrics/metrics.ts | 21 +- .../server/src/base/metrics/opentelemetry.ts | 79 +- .../backend/server/src/base/metrics/prisma.ts | 6 +- .../backend/server/src/base/nestjs/config.ts | 39 - .../backend/server/src/base/nestjs/index.ts | 2 - .../server/src/base/nestjs/optional-module.ts | 72 - .../backend/server/src/base/prisma/config.ts | 30 +- .../backend/server/src/base/prisma/factory.ts | 25 + .../backend/server/src/base/prisma/index.ts | 17 +- .../backend/server/src/base/prisma/service.ts | 27 - .../backend/server/src/base/redis/config.ts | 53 +- .../server/src/base/redis/instances.ts | 20 +- .../backend/server/src/base/runtime/event.ts | 7 - .../backend/server/src/base/runtime/index.ts | 11 - .../server/src/base/runtime/service.ts | 258 --- .../backend/server/src/base/storage/config.ts | 75 - .../server/src/base/storage/factory.ts | 20 + .../backend/server/src/base/storage/index.ts | 28 +- .../server/src/base/storage/providers/fs.ts | 12 +- .../src/base/storage/providers/index.ts | 136 +- .../src/base/storage/providers/provider.ts | 3 - .../{plugins => base}/storage/providers/r2.ts | 12 +- .../{plugins => base}/storage/providers/s3.ts | 11 +- .../server/src/base/throttler/config.ts | 47 +- .../server/src/base/throttler/index.ts | 30 +- .../backend/server/src/base/utils/request.ts | 2 +- .../backend/server/src/base/utils/types.ts | 10 +- .../server/src/base/websocket/adapter.ts | 7 +- .../server/src/base/websocket/config.ts | 40 +- .../backend/server/src/config/affine.env.ts | 42 - .../backend/server/src/config/affine.self.ts | 95 - packages/backend/server/src/config/affine.ts | 168 -- .../backend/server/src/core/auth/config.ts | 115 +- .../server/src/core/auth/controller.ts | 16 +- .../backend/server/src/core/auth/service.ts | 6 +- .../src/core/config/__tests__/service.spec.ts | 136 ++ .../backend/server/src/core/config/config.ts | 61 +- .../backend/server/src/core/config/index.ts | 21 +- .../server/src/core/config/resolver.ts | 236 +-- .../server/src/core/config/server-feature.ts | 7 - .../backend/server/src/core/config/service.ts | 113 +- .../backend/server/src/core/config/types.ts | 11 +- .../doc-renderer/__tests__/controller.spec.ts | 29 +- .../src/core/doc-renderer/controller.ts | 23 +- .../doc-service/__tests__/controller.spec.ts | 6 +- .../server/src/core/doc-service/config.ts | 25 +- .../__tests__/reader-from-database.spec.ts | 6 +- .../doc/__tests__/reader-from-rpc.spec.ts | 65 +- .../backend/server/src/core/doc/config.ts | 53 +- .../backend/server/src/core/doc/options.ts | 22 +- .../backend/server/src/core/doc/reader.ts | 6 +- .../server/src/core/features/service.ts | 10 +- .../backend/server/src/core/features/types.ts | 7 +- packages/backend/server/src/core/index.ts | 1 + .../backend/server/src/core/mail/config.ts | 62 +- .../backend/server/src/core/mail/sender.ts | 50 +- .../server/src/core/notification/service.ts | 12 +- .../server/src/core/selfhost/controller.ts | 21 +- .../backend/server/src/core/selfhost/guard.ts | 18 + .../backend/server/src/core/selfhost/index.ts | 3 +- .../server/src/core/selfhost/static.ts | 8 +- .../backend/server/src/core/storage/config.ts | 66 +- .../src/core/storage/wrappers/avatar.ts | 23 +- .../server/src/core/storage/wrappers/blob.ts | 17 +- .../backend/server/src/core/sync/gateway.ts | 33 +- .../server/src/core/user/controller.ts | 2 +- .../backend/server/src/core/user/event.ts | 61 - .../backend/server/src/core/user/index.ts | 8 +- .../backend/server/src/core/version/config.ts | 10 +- .../backend/server/src/core/version/guard.ts | 6 +- .../server/src/core/version/service.ts | 8 +- packages/backend/server/src/env.ts | 134 ++ packages/backend/server/src/global.d.ts | 19 +- packages/backend/server/src/index.ts | 12 +- .../server/src/mails/components/template.tsx | 2 +- packages/backend/server/src/mails/index.tsx | 4 +- packages/backend/server/src/models/base.ts | 4 - packages/backend/server/src/models/config.ts | 24 + packages/backend/server/src/models/feature.ts | 2 +- packages/backend/server/src/models/index.ts | 2 + packages/backend/server/src/models/session.ts | 6 +- .../server/src/plugins/captcha/config.ts | 44 +- .../server/src/plugins/captcha/guard.ts | 8 +- .../server/src/plugins/captcha/index.ts | 12 +- .../server/src/plugins/captcha/service.ts | 37 +- packages/backend/server/src/plugins/config.ts | 30 - .../server/src/plugins/copilot/config.ts | 88 +- .../server/src/plugins/copilot/context/job.ts | 27 +- .../src/plugins/copilot/context/service.ts | 8 +- .../server/src/plugins/copilot/controller.ts | 22 +- .../server/src/plugins/copilot/index.ts | 50 +- .../src/plugins/copilot/prompt/chat-prompt.ts | 8 +- .../src/plugins/copilot/prompt/prompts.ts | 2 +- .../src/plugins/copilot/prompt/service.ts | 8 +- .../src/plugins/copilot/providers/factory.ts | 107 + .../src/plugins/copilot/providers/fal.ts | 48 +- .../providers/{google.ts => gemini.ts} | 49 +- .../src/plugins/copilot/providers/index.ts | 211 +- .../src/plugins/copilot/providers/openai.ts | 54 +- .../plugins/copilot/providers/perplexity.ts | 38 +- .../src/plugins/copilot/providers/provider.ts | 45 + .../src/plugins/copilot/providers/types.ts | 170 ++ .../server/src/plugins/copilot/session.ts | 17 +- .../server/src/plugins/copilot/storage.ts | 6 +- .../src/plugins/copilot/transcript/service.ts | 12 +- .../server/src/plugins/copilot/types.ts | 196 +- .../copilot/workflow/executor/chat-image.ts | 11 +- .../copilot/workflow/executor/chat-text.ts | 11 +- .../copilot/workflow/executor/types.ts | 2 +- .../src/plugins/copilot/workflow/node.ts | 2 +- .../src/plugins/copilot/workflow/service.ts | 2 +- .../src/plugins/copilot/workflow/workflow.ts | 2 +- .../server/src/plugins/customerio/config.ts | 22 + .../server/src/plugins/customerio/index.ts | 10 + .../server/src/plugins/customerio/service.ts | 72 + .../server/src/plugins/gcloud/config.ts | 14 - .../server/src/plugins/gcloud/index.ts | 8 +- .../src/plugins/gcloud/logging/index.ts | 11 +- .../src/plugins/gcloud/logging/service.ts | 2 +- .../server/src/plugins/gcloud/metrics.ts | 14 +- packages/backend/server/src/plugins/index.ts | 13 - .../server/src/plugins/license/index.ts | 5 +- .../server/src/plugins/license/resolver.ts | 23 +- .../server/src/plugins/license/service.ts | 43 +- .../server/src/plugins/oauth/config.ts | 64 +- .../server/src/plugins/oauth/controller.ts | 2 +- .../server/src/plugins/oauth/factory.ts | 35 + .../backend/server/src/plugins/oauth/index.ts | 14 +- .../server/src/plugins/oauth/providers/def.ts | 37 + .../src/plugins/oauth/providers/github.ts | 11 +- .../src/plugins/oauth/providers/google.ts | 11 +- .../src/plugins/oauth/providers/oidc.ts | 35 +- .../server/src/plugins/oauth/register.ts | 58 - .../server/src/plugins/oauth/resolver.ts | 2 +- .../server/src/plugins/oauth/service.ts | 2 +- .../server/src/plugins/payment/config.ts | 40 +- .../server/src/plugins/payment/controller.ts | 19 +- .../server/src/plugins/payment/index.ts | 18 +- .../src/plugins/payment/license/controller.ts | 23 +- .../src/plugins/payment/manager/common.ts | 9 +- .../src/plugins/payment/manager/selfhost.ts | 6 +- .../src/plugins/payment/manager/user.ts | 11 +- .../src/plugins/payment/manager/workspace.ts | 6 +- .../server/src/plugins/payment/schedule.ts | 19 +- .../server/src/plugins/payment/service.ts | 94 +- .../server/src/plugins/payment/stripe.ts | 134 +- .../server/src/plugins/payment/webhook.ts | 7 +- .../backend/server/src/plugins/registry.ts | 30 - .../server/src/plugins/storage/config.ts | 27 - .../server/src/plugins/storage/index.ts | 42 - .../server/src/plugins/worker/config.ts | 17 +- .../server/src/plugins/worker/controller.ts | 21 +- .../server/src/plugins/worker/index.ts | 10 +- .../server/src/plugins/worker/service.ts | 31 + packages/backend/server/src/prelude.ts | 73 +- packages/backend/server/src/schema.gql | 61 +- .../graphql/src/graphql/admin/config.gql | 3 + .../admin/get-server-runtime-config.gql | 11 - .../admin/get-server-service-configs.gql | 6 - .../src/graphql/admin/update-config.gql | 3 + .../admin/update-server-runtime-configs.gql | 6 - packages/common/graphql/src/graphql/index.ts | 48 +- packages/common/graphql/src/schema.ts | 143 +- packages/frontend/admin/package.json | 3 + packages/frontend/admin/src/config.json | 350 ++++ .../admin/src/modules/config/index.tsx | 160 -- .../config/use-server-service-configs.ts | 62 - .../src/modules/nav/collapsible-item.tsx | 37 +- .../frontend/admin/src/modules/nav/nav.tsx | 13 +- .../admin/src/modules/nav/server-version.tsx | 4 +- .../admin/src/modules/nav/settings-item.tsx | 34 +- .../src/modules/settings/config-input.tsx | 125 ++ .../admin/src/modules/settings/config.ts | 31 + .../src/modules/settings/confirm-changes.tsx | 39 +- .../admin/src/modules/settings/index.tsx | 209 +- .../modules/settings/runtime-setting-row.tsx | 15 +- .../src/modules/settings/use-app-config.ts | 75 + .../settings/use-get-server-runtime-config.ts | 57 - .../use-update-server-runtime-config.ts | 41 - .../admin/src/modules/settings/utils.tsx | 68 - packages/frontend/admin/tsconfig.json | 4 +- packages/frontend/i18n/src/i18n.gen.ts | 4 + packages/frontend/i18n/src/resources/en.json | 3 +- tools/utils/src/workspace.gen.ts | 1 + yarn.lock | 13 + 274 files changed, 7583 insertions(+), 5841 deletions(-) create mode 100644 .docker/selfhost/schema.json delete mode 100644 .github/helm/affine/charts/graphql/templates/captcha-secret.yaml delete mode 100644 .github/helm/affine/charts/graphql/templates/copilot-secret.yaml delete mode 100644 .github/helm/affine/charts/graphql/templates/mailer.yaml delete mode 100644 .github/helm/affine/charts/graphql/templates/metrics-secret.yaml delete mode 100644 .github/helm/affine/charts/graphql/templates/oauth.yaml delete mode 100644 .github/helm/affine/charts/graphql/templates/payment.yml delete mode 100644 .github/helm/affine/templates/r2-secret.yaml create mode 100644 packages/backend/server/migrations/20250325075341_new_app_config/migration.sql create mode 100644 packages/backend/server/schema.gql create mode 100644 packages/backend/server/scripts/genconfig.ts delete mode 100644 packages/backend/server/src/__tests__/app/doc.e2e.ts delete mode 100644 packages/backend/server/src/__tests__/app/graphql.e2e.ts delete mode 100644 packages/backend/server/src/__tests__/app/renderer.e2e.ts delete mode 100644 packages/backend/server/src/__tests__/app/sync.e2e.ts delete mode 100644 packages/backend/server/src/__tests__/config.spec.ts rename packages/backend/server/src/__tests__/e2e/{ => apps}/app.spec.ts (77%) create mode 100644 packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts create mode 100644 packages/backend/server/src/__tests__/env.spec.ts create mode 100644 packages/backend/server/src/__tests__/mocks/copilot.mock.ts create mode 100644 packages/backend/server/src/__tests__/mocks/eventbus.mock.ts delete mode 100644 packages/backend/server/src/__tests__/utils/common.ts create mode 100644 packages/backend/server/src/base/config/__tests__/config.spec.ts create mode 100644 packages/backend/server/src/base/config/config.ts delete mode 100644 packages/backend/server/src/base/config/def.ts delete mode 100644 packages/backend/server/src/base/config/default.ts create mode 100644 packages/backend/server/src/base/config/factory.ts delete mode 100644 packages/backend/server/src/base/nestjs/config.ts delete mode 100644 packages/backend/server/src/base/nestjs/optional-module.ts create mode 100644 packages/backend/server/src/base/prisma/factory.ts delete mode 100644 packages/backend/server/src/base/prisma/service.ts delete mode 100644 packages/backend/server/src/base/runtime/event.ts delete mode 100644 packages/backend/server/src/base/runtime/index.ts delete mode 100644 packages/backend/server/src/base/runtime/service.ts delete mode 100644 packages/backend/server/src/base/storage/config.ts create mode 100644 packages/backend/server/src/base/storage/factory.ts rename packages/backend/server/src/{plugins => base}/storage/providers/r2.ts (62%) rename packages/backend/server/src/{plugins => base}/storage/providers/s3.ts (97%) delete mode 100644 packages/backend/server/src/config/affine.env.ts delete mode 100644 packages/backend/server/src/config/affine.self.ts delete mode 100644 packages/backend/server/src/config/affine.ts create mode 100644 packages/backend/server/src/core/config/__tests__/service.spec.ts delete mode 100644 packages/backend/server/src/core/config/server-feature.ts create mode 100644 packages/backend/server/src/core/index.ts create mode 100644 packages/backend/server/src/core/selfhost/guard.ts delete mode 100644 packages/backend/server/src/core/user/event.ts create mode 100644 packages/backend/server/src/env.ts create mode 100644 packages/backend/server/src/models/config.ts delete mode 100644 packages/backend/server/src/plugins/config.ts create mode 100644 packages/backend/server/src/plugins/copilot/providers/factory.ts rename packages/backend/server/src/plugins/copilot/providers/{google.ts => gemini.ts} (87%) create mode 100644 packages/backend/server/src/plugins/copilot/providers/provider.ts create mode 100644 packages/backend/server/src/plugins/copilot/providers/types.ts create mode 100644 packages/backend/server/src/plugins/customerio/config.ts create mode 100644 packages/backend/server/src/plugins/customerio/index.ts create mode 100644 packages/backend/server/src/plugins/customerio/service.ts delete mode 100644 packages/backend/server/src/plugins/gcloud/config.ts delete mode 100644 packages/backend/server/src/plugins/index.ts create mode 100644 packages/backend/server/src/plugins/oauth/factory.ts delete mode 100644 packages/backend/server/src/plugins/oauth/register.ts delete mode 100644 packages/backend/server/src/plugins/registry.ts delete mode 100644 packages/backend/server/src/plugins/storage/config.ts delete mode 100644 packages/backend/server/src/plugins/storage/index.ts create mode 100644 packages/backend/server/src/plugins/worker/service.ts create mode 100644 packages/common/graphql/src/graphql/admin/config.gql delete mode 100644 packages/common/graphql/src/graphql/admin/get-server-runtime-config.gql delete mode 100644 packages/common/graphql/src/graphql/admin/get-server-service-configs.gql create mode 100644 packages/common/graphql/src/graphql/admin/update-config.gql delete mode 100644 packages/common/graphql/src/graphql/admin/update-server-runtime-configs.gql create mode 100644 packages/frontend/admin/src/config.json delete mode 100644 packages/frontend/admin/src/modules/config/use-server-service-configs.ts create mode 100644 packages/frontend/admin/src/modules/settings/config-input.tsx create mode 100644 packages/frontend/admin/src/modules/settings/config.ts create mode 100644 packages/frontend/admin/src/modules/settings/use-app-config.ts delete mode 100644 packages/frontend/admin/src/modules/settings/use-get-server-runtime-config.ts delete mode 100644 packages/frontend/admin/src/modules/settings/use-update-server-runtime-config.ts diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json new file mode 100644 index 0000000000..130a0479fb --- /dev/null +++ b/.docker/selfhost/schema.json @@ -0,0 +1,900 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AFFiNE Application Configuration", + "type": "object", + "properties": { + "redis": { + "type": "object", + "description": "Configuration for redis module", + "properties": { + "db": { + "type": "number", + "description": "The database index of redis server to be used(Must be less than 10).\n@default 0\n@environment `REDIS_DATABASE`", + "default": 0 + }, + "host": { + "type": "string", + "description": "The host of the redis server.\n@default \"localhost\"\n@environment `REDIS_HOST`", + "default": "localhost" + }, + "port": { + "type": "number", + "description": "The port of the redis server.\n@default 6379\n@environment `REDIS_PORT`", + "default": 6379 + }, + "username": { + "type": "string", + "description": "The username of the redis server.\n@default \"\"\n@environment `REDIS_USERNAME`", + "default": "" + }, + "password": { + "type": "string", + "description": "The password of the redis server.\n@default \"\"\n@environment `REDIS_PASSWORD`", + "default": "" + }, + "ioredis": { + "type": "object", + "description": "The config for the ioredis client.\n@default {}\n@link https://github.com/luin/ioredis", + "default": {} + } + } + }, + "metrics": { + "type": "object", + "description": "Configuration for metrics module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable metric and tracing collection\n@default false", + "default": false + } + } + }, + "graphql": { + "type": "object", + "description": "Configuration for graphql module", + "properties": { + "apolloDriverConfig": { + "type": "object", + "description": "The config for underlying nestjs GraphQL and apollo driver engine.\n@default {\"buildSchemaOptions\":{\"numberScalarMode\":\"integer\"},\"useGlobalPrefix\":true,\"playground\":true,\"introspection\":true,\"sortSchema\":true}\n@link https://docs.nestjs.com/graphql/quick-start", + "default": { + "buildSchemaOptions": { + "numberScalarMode": "integer" + }, + "useGlobalPrefix": true, + "playground": true, + "introspection": true, + "sortSchema": true + } + } + } + }, + "crypto": { + "type": "object", + "description": "Configuration for crypto module", + "properties": { + "privateKey": { + "type": "string", + "description": "The private key for used by the crypto module to create signed tokens or encrypt data.\n@default \"\"\n@environment `AFFINE_PRIVATE_KEY`", + "default": "" + } + } + }, + "job": { + "type": "object", + "description": "Configuration for job module", + "properties": { + "queue": { + "type": "object", + "description": "The config for job queues\n@default {\"prefix\":\"affine_job\",\"defaultJobOptions\":{\"attempts\":5,\"removeOnComplete\":true,\"removeOnFail\":{\"age\":86400,\"count\":500}}}\n@link https://api.docs.bullmq.io/interfaces/v5.QueueOptions.html", + "default": { + "prefix": "affine_job", + "defaultJobOptions": { + "attempts": 5, + "removeOnComplete": true, + "removeOnFail": { + "age": 86400, + "count": 500 + } + } + } + }, + "worker": { + "type": "object", + "description": "The config for job workers\n@default {\"defaultWorkerOptions\":{}}\n@link https://api.docs.bullmq.io/interfaces/v5.WorkerOptions.html", + "default": { + "defaultWorkerOptions": {} + } + }, + "queues.copilot": { + "type": "object", + "description": "The config for copilot job queue\n@default {\"concurrency\":1}", + "properties": { + "concurrency": { + "type": "number" + } + }, + "default": { + "concurrency": 1 + } + }, + "queues.doc": { + "type": "object", + "description": "The config for doc job queue\n@default {\"concurrency\":1}", + "properties": { + "concurrency": { + "type": "number" + } + }, + "default": { + "concurrency": 1 + } + }, + "queues.notification": { + "type": "object", + "description": "The config for notification job queue\n@default {\"concurrency\":10}", + "properties": { + "concurrency": { + "type": "number" + } + }, + "default": { + "concurrency": 10 + } + }, + "queues.nightly": { + "type": "object", + "description": "The config for nightly job queue\n@default {\"concurrency\":1}", + "properties": { + "concurrency": { + "type": "number" + } + }, + "default": { + "concurrency": 1 + } + } + } + }, + "throttle": { + "type": "object", + "description": "Configuration for throttle module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether the throttler is enabled.\n@default true", + "default": true + }, + "throttlers.default": { + "type": "object", + "description": "The config for the default throttler.\n@default {\"ttl\":60,\"limit\":120}", + "default": { + "ttl": 60, + "limit": 120 + } + }, + "throttlers.strict": { + "type": "object", + "description": "The config for the strict throttler.\n@default {\"ttl\":60,\"limit\":20}", + "default": { + "ttl": 60, + "limit": 20 + } + } + } + }, + "websocket": { + "type": "object", + "description": "Configuration for websocket module", + "properties": { + "transports": { + "type": "array", + "description": "The enabled transports for accepting websocket traffics.\n@default [\"websocket\",\"polling\"]\n@link https://docs.nestjs.com/websockets/gateways#transports", + "items": { + "type": "string", + "enum": [ + "websocket", + "polling" + ] + }, + "default": [ + "websocket", + "polling" + ] + }, + "maxHttpBufferSize": { + "type": "number", + "description": "How many bytes or characters a message can be, before closing the session (to avoid DoS).\n@default 100000000", + "default": 100000000 + } + } + }, + "db": { + "type": "object", + "description": "Configuration for db module", + "properties": { + "datasourceUrl": { + "type": "string", + "description": "The datasource url for the prisma client.\n@default \"postgresql://localhost:5432/affine\"\n@environment `DATABASE_URL`", + "default": "postgresql://localhost:5432/affine" + }, + "prisma": { + "type": "object", + "description": "The config for the prisma client.\n@default {}\n@link https://www.prisma.io/docs/reference/api-reference/prisma-client-reference", + "default": {} + } + } + }, + "auth": { + "type": "object", + "description": "Configuration for auth module", + "properties": { + "allowSignup": { + "type": "boolean", + "description": "Whether allow new registrations.\n@default true", + "default": true + }, + "requireEmailDomainVerification": { + "type": "boolean", + "description": "Whether require email domain record verification before accessing restricted resources.\n@default false", + "default": false + }, + "requireEmailVerification": { + "type": "boolean", + "description": "Whether require email verification before accessing restricted resources(not implemented).\n@default true", + "default": true + }, + "passwordRequirements": { + "type": "object", + "description": "The password strength requirements when set new password.\n@default {\"min\":8,\"max\":32}", + "properties": { + "min": { + "type": "number" + }, + "max": { + "type": "number" + } + }, + "default": { + "min": 8, + "max": 32 + } + }, + "session.ttl": { + "type": "number", + "description": "Application auth expiration time in seconds.\n@default 1296000", + "default": 1296000 + }, + "session.ttr": { + "type": "number", + "description": "Application auth time to refresh in seconds.\n@default 604800", + "default": 604800 + } + } + }, + "mailer": { + "type": "object", + "description": "Configuration for mailer module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether enabled mail service.\n@default false", + "default": false + }, + "SMTP.host": { + "type": "string", + "description": "Host of the email server (e.g. smtp.gmail.com)\n@default \"\"\n@environment `MAILER_HOST`", + "default": "" + }, + "SMTP.port": { + "type": "number", + "description": "Port of the email server (they commonly are 25, 465 or 587)\n@default 465\n@environment `MAILER_PORT`", + "default": 465 + }, + "SMTP.username": { + "type": "string", + "description": "Username used to authenticate the email server\n@default \"\"\n@environment `MAILER_USER`", + "default": "" + }, + "SMTP.password": { + "type": "string", + "description": "Password used to authenticate the email server\n@default \"\"\n@environment `MAILER_PASSWORD`", + "default": "" + }, + "SMTP.sender": { + "type": "string", + "description": "Sender of all the emails (e.g. \"AFFiNE Team \")\n@default \"\"\n@environment `MAILER_SENDER`", + "default": "" + }, + "SMTP.ignoreTLS": { + "type": "boolean", + "description": "Whether ignore email server's TSL certification verification. Enable it for self-signed certificates.\n@default false\n@environment `MAILER_IGNORE_TLS`", + "default": false + } + } + }, + "doc": { + "type": "object", + "description": "Configuration for doc module", + "properties": { + "experimental.yocto": { + "type": "boolean", + "description": "Use `y-octo` to merge updates at the same time when merging using Yjs.\n@default false", + "default": false + }, + "history.interval": { + "type": "number", + "description": "The minimum time interval in milliseconds of creating a new history snapshot when doc get updated.\n@default 600000", + "default": 600000 + } + } + }, + "storages": { + "type": "object", + "description": "Configuration for storages module", + "properties": { + "avatar.publicPath": { + "type": "string", + "description": "The public accessible path prefix for user avatars.\n@default \"/api/avatars/\"", + "default": "/api/avatars/" + }, + "avatar.storage": { + "type": "object", + "description": "The config of storage for user avatars.\n@default {\"provider\":\"fs\",\"bucket\":\"avatars\",\"config\":{\"path\":\"~/.affine/storage\"}}", + "oneOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "fs" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "aws-s3" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "description": "The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + "properties": { + "credentials": { + "type": "object", + "description": "The credentials for the s3 compatible storage provider.", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "cloudflare-r2" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "description": "The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + "properties": { + "credentials": { + "type": "object", + "description": "The credentials for the s3 compatible storage provider.", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + }, + "accountId": { + "type": "string", + "description": "The account id for the cloudflare r2 storage provider." + } + } + } + } + } + ], + "default": { + "provider": "fs", + "bucket": "avatars", + "config": { + "path": "~/.affine/storage" + } + } + }, + "blob.storage": { + "type": "object", + "description": "The config of storage for all uploaded blobs(images, videos, etc.).\n@default {\"provider\":\"fs\",\"bucket\":\"blobs\",\"config\":{\"path\":\"~/.affine/storage\"}}", + "oneOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "fs" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "aws-s3" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "description": "The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + "properties": { + "credentials": { + "type": "object", + "description": "The credentials for the s3 compatible storage provider.", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "cloudflare-r2" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "description": "The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + "properties": { + "credentials": { + "type": "object", + "description": "The credentials for the s3 compatible storage provider.", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + }, + "accountId": { + "type": "string", + "description": "The account id for the cloudflare r2 storage provider." + } + } + } + } + } + ], + "default": { + "provider": "fs", + "bucket": "blobs", + "config": { + "path": "~/.affine/storage" + } + } + } + } + }, + "server": { + "type": "object", + "description": "Configuration for server module", + "properties": { + "name": { + "type": "string", + "description": "A recognizable name for the server. Will be shown when connected with AFFiNE Desktop.\n@default \"AFFiNE Cloud\"", + "default": "AFFiNE Cloud" + }, + "externalUrl": { + "type": "string", + "description": "Base url of AFFiNE server, used for generating external urls.\nDefault to be `[server.protocol]://[server.host][:server.port]` if not specified.\n \n@default \"http://localhost:3010\"\n@environment `AFFINE_SERVER_EXTERNAL_URL`", + "default": "http://localhost:3010" + }, + "https": { + "type": "boolean", + "description": "Whether the server is hosted on a ssl enabled domain (https://).\n@default false\n@environment `AFFINE_SERVER_HTTPS`", + "default": false + }, + "host": { + "type": "string", + "description": "Where the server get deployed(FQDN).\n@default \"localhost\"\n@environment `AFFINE_SERVER_HOST`", + "default": "localhost" + }, + "port": { + "type": "number", + "description": "Which port the server will listen on.\n@default 3010\n@environment `AFFINE_SERVER_PORT`", + "default": 3010 + }, + "path": { + "type": "string", + "description": "Subpath where the server get deployed if there is.\n@default \"\"\n@environment `AFFINE_SERVER_SUB_PATH`", + "default": "" + } + } + }, + "flags": { + "type": "object", + "description": "Configuration for flags module", + "properties": { + "earlyAccessControl": { + "type": "boolean", + "description": "Only allow users with early access features to access the app\n@default false", + "default": false + } + } + }, + "client": { + "type": "object", + "description": "Configuration for client module", + "properties": { + "versionControl.enabled": { + "type": "boolean", + "description": "Whether check version of client before accessing the server.\n@default false", + "default": false + }, + "versionControl.requiredVersion": { + "type": "string", + "description": "Allowed version range of the app that allowed to access the server. Requires 'client/versionControl.enabled' to be true to take effect.\n@default \">=0.20.0\"", + "default": ">=0.20.0" + } + } + }, + "captcha": { + "type": "object", + "description": "Configuration for captcha module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Check captcha challenge when user authenticating the app.\n@default false", + "default": false + }, + "config": { + "type": "object", + "description": "The config for the captcha plugin.\n@default {\"turnstile\":{\"secret\":\"\"},\"challenge\":{\"bits\":20}}", + "default": { + "turnstile": { + "secret": "" + }, + "challenge": { + "bits": 20 + } + } + } + } + }, + "copilot": { + "type": "object", + "description": "Configuration for copilot module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether to enable the copilot plugin.\n@default false", + "default": false + }, + "providers.openai": { + "type": "object", + "description": "The config for the openai provider.\n@default {\"apiKey\":\"\"}\n@link https://github.com/openai/openai-node", + "default": { + "apiKey": "" + } + }, + "providers.fal": { + "type": "object", + "description": "The config for the fal provider.\n@default {\"apiKey\":\"\"}", + "default": { + "apiKey": "" + } + }, + "providers.gemini": { + "type": "object", + "description": "The config for the gemini provider.\n@default {\"apiKey\":\"\"}", + "default": { + "apiKey": "" + } + }, + "providers.perplexity": { + "type": "object", + "description": "The config for the perplexity provider.\n@default {\"apiKey\":\"\"}", + "default": { + "apiKey": "" + } + }, + "unsplash": { + "type": "object", + "description": "The config for the unsplash key.\n@default {\"key\":\"\"}", + "default": { + "key": "" + } + }, + "storage": { + "type": "object", + "description": "The config for the storage provider.\n@default {\"provider\":\"fs\",\"bucket\":\"copilot\",\"config\":{\"path\":\"~/.affine/storage\"}}", + "oneOf": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "fs" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + } + } + } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "aws-s3" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "description": "The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + "properties": { + "credentials": { + "type": "object", + "description": "The credentials for the s3 compatible storage provider.", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "provider": { + "type": "string", + "enum": [ + "cloudflare-r2" + ] + }, + "bucket": { + "type": "string" + }, + "config": { + "type": "object", + "description": "The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html", + "properties": { + "credentials": { + "type": "object", + "description": "The credentials for the s3 compatible storage provider.", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + } + }, + "accountId": { + "type": "string", + "description": "The account id for the cloudflare r2 storage provider." + } + } + } + } + } + ], + "default": { + "provider": "fs", + "bucket": "copilot", + "config": { + "path": "~/.affine/storage" + } + } + } + } + }, + "customerIo": { + "type": "object", + "description": "Configuration for customerIo module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable customer.io integration\n@default false", + "default": false + }, + "token": { + "type": "string", + "description": "Customer.io token\n@default \"\"", + "default": "" + } + } + }, + "oauth": { + "type": "object", + "description": "Configuration for oauth module", + "properties": { + "providers.google": { + "type": "object", + "description": "Google OAuth provider config\n@default {\"clientId\":\"\",\"clientSecret\":\"\"}\n@link https://developers.google.com/identity/protocols/oauth2/web-server", + "properties": { + "clientId": { + "type": "string" + }, + "clientSecret": { + "type": "string" + }, + "args": { + "type": "object" + } + }, + "default": { + "clientId": "", + "clientSecret": "" + } + }, + "providers.github": { + "type": "object", + "description": "GitHub OAuth provider config\n@default {\"clientId\":\"\",\"clientSecret\":\"\"}\n@link https://docs.github.com/en/apps/oauth-apps", + "properties": { + "clientId": { + "type": "string" + }, + "clientSecret": { + "type": "string" + }, + "args": { + "type": "object" + } + }, + "default": { + "clientId": "", + "clientSecret": "" + } + }, + "providers.oidc": { + "type": "object", + "description": "OIDC OAuth provider config\n@default {\"clientId\":\"\",\"clientSecret\":\"\",\"issuer\":\"\",\"args\":{}}", + "default": { + "clientId": "", + "clientSecret": "", + "issuer": "", + "args": {} + } + } + } + }, + "payment": { + "type": "object", + "description": "Configuration for payment module", + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether enable payment plugin\n@default false", + "default": false + }, + "showLifetimePrice": { + "type": "boolean", + "description": "Whether enable lifetime price and allow user to pay for it.\n@default true", + "default": true + }, + "apiKey": { + "type": "string", + "description": "Stripe API key to enable payment service.\n@default \"\"\n@environment `STRIPE_API_KEY`", + "default": "" + }, + "webhookKey": { + "type": "string", + "description": "Stripe webhook key to enable payment service.\n@default \"\"\n@environment `STRIPE_WEBHOOK_KEY`", + "default": "" + }, + "stripe": { + "type": "object", + "description": "Stripe API keys\n@default {}\n@link https://docs.stripe.com/api", + "default": {} + } + } + }, + "worker": { + "type": "object", + "description": "Configuration for worker module", + "properties": { + "allowedOrigin": { + "type": "array", + "description": "Allowed origin\n@default [\"localhost\",\"127.0.0.1\"]", + "default": [ + "localhost", + "127.0.0.1" + ] + } + } + } + } +} \ No newline at end of file diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs index e4cfbd99a1..20c9ee726f 100644 --- a/.github/actions/deploy/deploy.mjs +++ b/.github/actions/deploy/deploy.mjs @@ -10,29 +10,10 @@ const { DATABASE_USERNAME, DATABASE_PASSWORD, DATABASE_NAME, - R2_ACCOUNT_ID, - R2_ACCESS_KEY_ID, - R2_SECRET_ACCESS_KEY, - CAPTCHA_TURNSTILE_SECRET, - METRICS_CUSTOMER_IO_TOKEN, - COPILOT_OPENAI_API_KEY, - COPILOT_FAL_API_KEY, - COPILOT_GOOGLE_API_KEY, - COPILOT_PERPLEXITY_API_KEY, - COPILOT_UNSPLASH_API_KEY, - MAILER_SENDER, - MAILER_USER, - MAILER_PASSWORD, - AFFINE_GOOGLE_CLIENT_ID, - AFFINE_GOOGLE_CLIENT_SECRET, CLOUD_SQL_IAM_ACCOUNT, APP_IAM_ACCOUNT, - GCLOUD_CONNECTION_NAME, - GCLOUD_CLOUD_SQL_INTERNAL_ENDPOINT, REDIS_HOST, REDIS_PASSWORD, - STRIPE_API_KEY, - STRIPE_WEBHOOK_KEY, STATIC_IP_NAME, } = process.env; @@ -89,13 +70,11 @@ const createHelmCommand = ({ isDryRun }) => { const redisAndPostgres = isProduction || isBeta || isInternal ? [ - `--set-string global.database.url=${DATABASE_URL}`, + `--set cloud-sql-proxy.enabled=true`, + `--set-string global.database.host=${DATABASE_URL}`, `--set-string global.database.user=${DATABASE_USERNAME}`, `--set-string global.database.password=${DATABASE_PASSWORD}`, `--set-string global.database.name=${DATABASE_NAME}`, - `--set global.database.gcloud.enabled=true`, - `--set-string global.database.gcloud.connectionName="${GCLOUD_CONNECTION_NAME}"`, - `--set-string global.database.gcloud.cloudSqlInternal="${GCLOUD_CLOUD_SQL_INTERNAL_ENDPOINT}"`, `--set-string global.redis.host="${REDIS_HOST}"`, `--set-string global.redis.password="${REDIS_PASSWORD}"`, ] @@ -141,14 +120,12 @@ const createHelmCommand = ({ isDryRun }) => { const deployCommand = [ `helm upgrade --install affine .github/helm/affine`, `--namespace ${namespace}`, + `--set-string global.deployment.type="affine"`, + `--set-string global.deployment.platform="gcp"`, `--set-string global.app.buildType="${buildType}"`, `--set global.ingress.enabled=true`, `--set-json global.ingress.annotations="{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${STATIC_IP_NAME}\\" }"`, `--set-string global.ingress.host="${host}"`, - `--set global.objectStorage.r2.enabled=true`, - `--set-string global.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`, - `--set-string global.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`, - `--set-string global.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`, `--set-string global.version="${APP_VERSION}"`, ...redisAndPostgres, `--set web.replicaCount=${replica.web}`, @@ -156,27 +133,6 @@ const createHelmCommand = ({ isDryRun }) => { `--set graphql.replicaCount=${replica.graphql}`, `--set-string graphql.image.tag="${imageTag}"`, `--set graphql.app.host=${host}`, - `--set graphql.app.captcha.enabled=true`, - `--set-string graphql.app.captcha.turnstile.secret="${CAPTCHA_TURNSTILE_SECRET}"`, - `--set graphql.app.copilot.enabled=true`, - `--set-string graphql.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`, - `--set-string graphql.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`, - `--set-string graphql.app.copilot.google.key="${COPILOT_GOOGLE_API_KEY}"`, - `--set-string graphql.app.copilot.perplexity.key="${COPILOT_PERPLEXITY_API_KEY}"`, - `--set-string graphql.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`, - `--set-string graphql.app.mailer.sender="${MAILER_SENDER}"`, - `--set-string graphql.app.mailer.user="${MAILER_USER}"`, - `--set-string graphql.app.mailer.password="${MAILER_PASSWORD}"`, - `--set-string graphql.app.oauth.google.enabled=true`, - `--set-string graphql.app.oauth.google.clientId="${AFFINE_GOOGLE_CLIENT_ID}"`, - `--set-string graphql.app.oauth.google.clientSecret="${AFFINE_GOOGLE_CLIENT_SECRET}"`, - `--set-string graphql.app.payment.stripe.apiKey="${STRIPE_API_KEY}"`, - `--set-string graphql.app.payment.stripe.webhookKey="${STRIPE_WEBHOOK_KEY}"`, - `--set graphql.app.metrics.enabled=true`, - `--set-string graphql.app.metrics.customerIo.token="${METRICS_CUSTOMER_IO_TOKEN}"`, - `--set graphql.app.experimental.enableJwstCodec=${namespace === 'dev'}`, - `--set graphql.app.features.earlyAccessPreview=false`, - `--set graphql.app.features.syncClientVersionCheck=true`, `--set sync.replicaCount=${replica.sync}`, `--set-string sync.image.tag="${imageTag}"`, `--set-string renderer.image.tag="${imageTag}"`, @@ -184,11 +140,6 @@ const createHelmCommand = ({ isDryRun }) => { `--set renderer.replicaCount=${replica.renderer}`, `--set-string doc.image.tag="${imageTag}"`, `--set doc.app.host=${host}`, - `--set doc.app.copilot.enabled=true`, - `--set-string doc.app.copilot.openai.key="${COPILOT_OPENAI_API_KEY}"`, - `--set-string doc.app.copilot.fal.key="${COPILOT_FAL_API_KEY}"`, - `--set-string doc.app.copilot.perplexity.key="${COPILOT_PERPLEXITY_API_KEY}"`, - `--set-string doc.app.copilot.unsplash.key="${COPILOT_UNSPLASH_API_KEY}"`, `--set doc.replicaCount=${replica.doc}`, ...serviceAnnotations, ...resources, diff --git a/.github/helm/affine/charts/doc/templates/deployment.yaml b/.github/helm/affine/charts/doc/templates/deployment.yaml index 9b3ff410b5..3dd107ed53 100644 --- a/.github/helm/affine/charts/doc/templates/deployment.yaml +++ b/.github/helm/affine/charts/doc/templates/deployment.yaml @@ -40,7 +40,9 @@ spec: - name: NO_COLOR value: "1" - name: DEPLOYMENT_TYPE - value: "affine" + value: "{{ .Values.global.deployment.type }}" + - name: DEPLOYMENT_PLATFORM + value: "{{ .Values.global.deployment.platform }}" - name: SERVER_FLAVOR value: "doc" - name: AFFINE_ENV @@ -75,50 +77,6 @@ spec: value: "{{ .Values.app.host }}" - name: AFFINE_SERVER_HTTPS value: "{{ .Values.app.https }}" - {{ if .Values.global.objectStorage.r2.enabled }} - - name: R2_OBJECT_STORAGE_ACCOUNT_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accountId - - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accessKeyId - - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: secretAccessKey - {{ end }} - {{ if .Values.app.copilot.enabled }} - - name: COPILOT_OPENAI_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: openaiSecret - - name: COPILOT_FAL_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: falSecret - - name: COPILOT_GOOGLE_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: googleSecret - - name: COPILOT_PERPLEXITY_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: perplexitySecret - - name: COPILOT_UNSPLASH_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: unsplashSecret - {{ end }} ports: - name: http containerPort: {{ .Values.global.docService.port }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml index 26d7832554..a94f5cc72f 100644 --- a/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml @@ -1,4 +1,4 @@ -{{- if .Values.global.database.gcloud.enabled -}} +{{- if .Values.enabled -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -42,7 +42,7 @@ spec: - "0.0.0.0" - "--structured-logs" - "--auto-iam-authn" - - "{{ .Values.global.database.gcloud.connectionName }}" + - "{{ .Values.database.connectionName }}" env: # Enable HTTP healthchecks on port 9801. This enables /liveness, # /readiness and /startup health check endpoints. Allow connections @@ -56,7 +56,7 @@ spec: value: 0.0.0.0 ports: - name: cloud-sql-proxy - containerPort: {{ .Values.global.database.gcloud.proxyPort }} + containerPort: {{ .Values.service.port }} protocol: TCP - containerPort: 9801 protocol: TCP diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml index 947365fe35..d682ce2823 100644 --- a/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml +++ b/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml @@ -1,4 +1,5 @@ replicaCount: 3 +enabled: false image: # the tag is defined as chart appVersion. diff --git a/.github/helm/affine/charts/graphql/templates/captcha-secret.yaml b/.github/helm/affine/charts/graphql/templates/captcha-secret.yaml deleted file mode 100644 index 009091c9c8..0000000000 --- a/.github/helm/affine/charts/graphql/templates/captcha-secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.app.captcha.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.captcha.secretName }}" -type: Opaque -data: - turnstileSecret: {{ .Values.app.captcha.turnstile.secret | b64enc }} -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/copilot-secret.yaml b/.github/helm/affine/charts/graphql/templates/copilot-secret.yaml deleted file mode 100644 index de20bf887c..0000000000 --- a/.github/helm/affine/charts/graphql/templates/copilot-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.app.copilot.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.copilot.secretName }}" -type: Opaque -data: - openaiSecret: {{ .Values.app.copilot.openai.key | b64enc }} - falSecret: {{ .Values.app.copilot.fal.key | b64enc }} - googleSecret: {{ .Values.app.copilot.google.key | b64enc }} - perplexitySecret: {{ .Values.app.copilot.perplexity.key | b64enc }} - unsplashSecret: {{ .Values.app.copilot.unsplash.key | b64enc }} -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index 47fac36b99..d2139e368b 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -40,7 +40,9 @@ spec: - name: NO_COLOR value: "1" - name: DEPLOYMENT_TYPE - value: "affine" + value: "{{ .Values.global.deployment.type }}" + - name: DEPLOYMENT_PLATFORM + value: "{{ .Values.global.deployment.platform }}" - name: SERVER_FLAVOR value: "graphql" - name: AFFINE_ENV @@ -52,8 +54,6 @@ spec: key: postgres-password - name: DATABASE_URL value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} - - name: REDIS_SERVER_ENABLED - value: "true" - name: REDIS_SERVER_HOST value: "{{ .Values.global.redis.host }}" - name: REDIS_SERVER_PORT @@ -75,135 +75,8 @@ spec: value: "{{ .Values.app.host }}" - name: AFFINE_SERVER_HTTPS value: "{{ .Values.app.https }}" - - name: ENABLE_R2_OBJECT_STORAGE - value: "{{ .Values.global.objectStorage.r2.enabled }}" - - name: FEATURES_SYNC_CLIENT_VERSION_CHECK - value: "{{ .Values.app.features.syncClientVersionCheck }}" - - name: MAILER_HOST - valueFrom: - secretKeyRef: - name: "{{ .Values.app.mailer.secretName }}" - key: host - - name: MAILER_PORT - valueFrom: - secretKeyRef: - name: "{{ .Values.app.mailer.secretName }}" - key: port - - name: MAILER_USER - valueFrom: - secretKeyRef: - name: "{{ .Values.app.mailer.secretName }}" - key: user - - name: MAILER_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Values.app.mailer.secretName }}" - key: password - - name: MAILER_SENDER - valueFrom: - secretKeyRef: - name: "{{ .Values.app.mailer.secretName }}" - key: sender - - name: STRIPE_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.payment.stripe.secretName }}" - key: stripeAPIKey - - name: STRIPE_WEBHOOK_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.payment.stripe.secretName }}" - key: stripeWebhookKey - name: DOC_SERVICE_ENDPOINT value: "http://{{ .Values.global.docService.name }}:{{ .Values.global.docService.port }}" - {{ if .Values.app.experimental.enableJwstCodec }} - - name: DOC_MERGE_USE_JWST_CODEC - value: "true" - {{ end }} - {{ if .Values.global.objectStorage.r2.enabled }} - - name: R2_OBJECT_STORAGE_ACCOUNT_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accountId - - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accessKeyId - - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: secretAccessKey - {{ end }} - {{ if .Values.app.captcha.enabled }} - - name: CAPTCHA_TURNSTILE_SECRET - valueFrom: - secretKeyRef: - name: "{{ .Values.app.captcha.secretName }}" - key: turnstileSecret - {{ end }} - {{ if .Values.app.copilot.enabled }} - - name: COPILOT_OPENAI_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: openaiSecret - - name: COPILOT_FAL_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: falSecret - - name: COPILOT_GOOGLE_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: googleSecret - - name: COPILOT_PERPLEXITY_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: perplexitySecret - - name: COPILOT_UNSPLASH_API_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.app.copilot.secretName }}" - key: unsplashSecret - {{ end }} - {{ if .Values.app.oauth.google.enabled }} - - name: OAUTH_GOOGLE_ENABLED - value: "true" - - name: OAUTH_GOOGLE_CLIENT_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.app.oauth.google.secretName }}" - key: clientId - - name: OAUTH_GOOGLE_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: "{{ .Values.app.oauth.google.secretName }}" - key: clientSecret - {{ end }} - {{ if .Values.app.oauth.github.enabled }} - - name: OAUTH_GITHUB_CLIENT_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.app.oauth.github.secretName }}" - key: clientId - - name: OAUTH_GITHUB_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: "{{ .Values.app.oauth.github.secretName }}" - key: clientSecret - {{ end }} - {{ if .Values.app.metrics.enabled }} - - name: METRICS_CUSTOMER_IO_TOKEN - valueFrom: - secretKeyRef: - name: "{{ .Values.app.metrics.secretName }}" - key: customerIoSecret - {{ end }} ports: - name: http containerPort: {{ .Values.service.port }} diff --git a/.github/helm/affine/charts/graphql/templates/mailer.yaml b/.github/helm/affine/charts/graphql/templates/mailer.yaml deleted file mode 100644 index 2ce5174441..0000000000 --- a/.github/helm/affine/charts/graphql/templates/mailer.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.app.mailer.secretName -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.mailer.secretName }}" -type: Opaque -data: - host: "{{ .Values.app.mailer.host | b64enc }}" - port: "{{ .Values.app.mailer.port | b64enc }}" - user: "{{ .Values.app.mailer.user | b64enc }}" - password: "{{ .Values.app.mailer.password | b64enc }}" - sender: "{{ .Values.app.mailer.sender | b64enc }}" -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/metrics-secret.yaml b/.github/helm/affine/charts/graphql/templates/metrics-secret.yaml deleted file mode 100644 index 16e03e0eae..0000000000 --- a/.github/helm/affine/charts/graphql/templates/metrics-secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.app.metrics.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.metrics.secretName }}" -type: Opaque -data: - customerIoSecret: {{ .Values.app.metrics.customerIo.token | b64enc }} -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/migration.yaml b/.github/helm/affine/charts/graphql/templates/migration.yaml index 32b88d0dd5..3d49d564fd 100644 --- a/.github/helm/affine/charts/graphql/templates/migration.yaml +++ b/.github/helm/affine/charts/graphql/templates/migration.yaml @@ -23,37 +23,27 @@ spec: - name: AFFINE_ENV value: "{{ .Release.Namespace }}" - name: DEPLOYMENT_TYPE - value: "affine" + value: "{{ .Values.global.deployment.type }}" + - name: DEPLOYMENT_PLATFORM + value: "{{ .Values.global.deployment.platform }}" - name: DATABASE_PASSWORD valueFrom: secretKeyRef: name: pg-postgresql key: postgres-password - {{ if not .Values.global.database.gcloud.enabled }} - name: DATABASE_URL value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} - {{ end }} - {{ if .Values.global.database.gcloud.enabled }} - - name: DATABASE_URL - value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} - {{ end }} - {{ if .Values.global.objectStorage.r2.enabled }} - - name: R2_OBJECT_STORAGE_ACCOUNT_ID + - name: REDIS_SERVER_HOST + value: "{{ .Values.global.redis.host }}" + - name: REDIS_SERVER_PORT + value: "{{ .Values.global.redis.port }}" + - name: REDIS_SERVER_USER + value: "{{ .Values.global.redis.username }}" + - name: REDIS_SERVER_PASSWORD valueFrom: secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accountId - - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accessKeyId - - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: secretAccessKey - {{ end }} + name: redis + key: redis-password resources: requests: cpu: '100m' diff --git a/.github/helm/affine/charts/graphql/templates/oauth.yaml b/.github/helm/affine/charts/graphql/templates/oauth.yaml deleted file mode 100644 index 08d8ccd0cc..0000000000 --- a/.github/helm/affine/charts/graphql/templates/oauth.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if .Values.app.oauth.google.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.oauth.google.secretName }}" -type: Opaque -data: - clientId: "{{ .Values.app.oauth.google.clientId | b64enc }}" - clientSecret: "{{ .Values.app.oauth.google.clientSecret | b64enc }}" -{{- end }} ---- -{{- if .Values.app.oauth.github.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.oauth.github.secretName }}" -type: Opaque -data: - clientId: "{{ .Values.app.oauth.github.clientId | b64enc }}" - clientSecret: "{{ .Values.app.oauth.github.clientSecret | b64enc }}" -{{- end }} diff --git a/.github/helm/affine/charts/graphql/templates/payment.yml b/.github/helm/affine/charts/graphql/templates/payment.yml deleted file mode 100644 index ec89e0201f..0000000000 --- a/.github/helm/affine/charts/graphql/templates/payment.yml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.app.payment.stripe.secretName }}" -type: Opaque -data: - stripeAPIKey: "{{ .Values.app.payment.stripe.apiKey | b64enc }}" - stripeWebhookKey: "{{ .Values.app.payment.stripe.webhookKey | b64enc }}" diff --git a/.github/helm/affine/charts/graphql/values.yaml b/.github/helm/affine/charts/graphql/values.yaml index fcaeb60690..9969ca69bc 100644 --- a/.github/helm/affine/charts/graphql/values.yaml +++ b/.github/helm/affine/charts/graphql/values.yaml @@ -10,55 +10,12 @@ fullnameOverride: '' # map to NODE_ENV environment variable env: 'production' app: - experimental: - enableJwstCodec: true # AFFINE_SERVER_SUB_PATH path: '' # AFFINE_SERVER_HOST host: '0.0.0.0' https: true - captcha: - enabled: false - secretName: captcha - turnstile: - secret: '' - copilot: - enabled: false - secretName: copilot - openai: - key: '' - oauth: - google: - enabled: false - secretName: oauth-google - clientId: '' - clientSecret: '' - github: - enabled: false - secretName: oauth-github - clientId: '' - clientSecret: '' - mailer: - secretName: 'mailer' - host: 'smtp.gmail.com' - port: '465' - user: '' - password: '' - sender: 'noreply@toeverything.info' - metrics: - enabled: false - secretName: 'metrics' - customerIo: - token: '' - payment: - stripe: - secretName: 'stripe' - apiKey: '' - webhookKey: '' - features: - earlyAccessPreview: false - syncClientVersionCheck: false - + serviceAccount: create: true annotations: {} diff --git a/.github/helm/affine/charts/renderer/templates/deployment.yaml b/.github/helm/affine/charts/renderer/templates/deployment.yaml index 912c24f5f8..0aa0a0315e 100644 --- a/.github/helm/affine/charts/renderer/templates/deployment.yaml +++ b/.github/helm/affine/charts/renderer/templates/deployment.yaml @@ -40,7 +40,9 @@ spec: - name: NO_COLOR value: "1" - name: DEPLOYMENT_TYPE - value: "affine" + value: "{{ .Values.global.deployment.type }}" + - name: DEPLOYMENT_PLATFORM + value: "{{ .Values.global.deployment.platform }}" - name: SERVER_FLAVOR value: "renderer" - name: AFFINE_ENV @@ -75,25 +77,6 @@ spec: value: "{{ .Values.app.host }}" - name: AFFINE_SERVER_HTTPS value: "{{ .Values.app.https }}" - - name: ENABLE_R2_OBJECT_STORAGE - value: "{{ .Values.global.objectStorage.r2.enabled }}" - {{ if .Values.global.objectStorage.r2.enabled }} - - name: R2_OBJECT_STORAGE_ACCOUNT_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accountId - - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: accessKeyId - - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: "{{ .Values.global.objectStorage.r2.secretName }}" - key: secretAccessKey - {{ end }} - name: DOC_SERVICE_ENDPOINT value: "http://{{ .Values.global.docService.name }}:{{ .Values.global.docService.port }}" ports: diff --git a/.github/helm/affine/charts/sync/templates/deployment.yaml b/.github/helm/affine/charts/sync/templates/deployment.yaml index e0f772a771..963fae19ac 100644 --- a/.github/helm/affine/charts/sync/templates/deployment.yaml +++ b/.github/helm/affine/charts/sync/templates/deployment.yaml @@ -42,7 +42,9 @@ spec: - name: NO_COLOR value: "1" - name: DEPLOYMENT_TYPE - value: "affine" + value: "{{ .Values.global.deployment.type }}" + - name: DEPLOYMENT_PLATFORM + value: "{{ .Values.global.deployment.platform }}" - name: SERVER_FLAVOR value: "sync" - name: AFFINE_ENV @@ -54,8 +56,6 @@ spec: key: postgres-password - name: DATABASE_URL value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.url }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} - - name: REDIS_SERVER_ENABLED - value: "true" - name: REDIS_SERVER_HOST value: "{{ .Values.global.redis.host }}" - name: REDIS_SERVER_PORT diff --git a/.github/helm/affine/templates/r2-secret.yaml b/.github/helm/affine/templates/r2-secret.yaml deleted file mode 100644 index d5c49fb2fb..0000000000 --- a/.github/helm/affine/templates/r2-secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.global.objectStorage.r2.enabled -}} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ .Values.global.objectStorage.r2.secretName }}" -type: Opaque -data: - accountId: {{ .Values.global.objectStorage.r2.accountId | b64enc }} - accessKeyId: {{ .Values.global.objectStorage.r2.accessKeyId | b64enc }} - secretAccessKey: {{ .Values.global.objectStorage.r2.secretAccessKey | b64enc }} -{{- end }} diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml index 671cef0887..cf826108e7 100644 --- a/.github/helm/affine/values.yaml +++ b/.github/helm/affine/values.yaml @@ -11,18 +11,10 @@ global: privateKey: '' database: user: 'postgres' - url: 'pg-postgresql' + host: 'pg-postgresql' port: '5432' name: 'affine' password: '' - gcloud: - enabled: false - # use for migration - cloudSqlInternal: '' - connectionName: '' - serviceAccount: '' - cloudProxyReplicas: 3 - proxyPort: '5432' redis: enabled: true host: 'redis-master' @@ -30,18 +22,13 @@ global: username: '' password: '' database: 0 - objectStorage: - r2: - enabled: false - secretName: r2 - accountId: '' - accessKeyId: '' - secretAccessKey: '' - gke: - enabled: true docService: name: 'affine-doc' port: 3020 + deployment: + # change to 'selfhosted' and 'unknown' if this chart is ready to be used for selfhosted deployment + type: 'affine' + platform: 'gcp' graphql: service: diff --git a/.prettierignore b/.prettierignore index b9f6741fe9..4adad258d6 100644 --- a/.prettierignore +++ b/.prettierignore @@ -37,3 +37,4 @@ packages/frontend/apps/android/App/** packages/frontend/apps/ios/App/** tests/blocksuite/snapshots blocksuite/docs/api/** +packages/frontend/admin/src/config.json diff --git a/oxlint.json b/oxlint.json index ac240d9ba8..5c0bf45dc7 100644 --- a/oxlint.json +++ b/oxlint.json @@ -37,7 +37,8 @@ "packages/frontend/apps/android/App/**", "packages/frontend/apps/ios/App/**", "tests/blocksuite/snapshots", - "blocksuite/docs/api/**" + "blocksuite/docs/api/**", + "packages/frontend/admin/src/config.json" ], "rules": { "no-await-in-loop": "allow", diff --git a/packages/backend/server/migrations/20250325075341_new_app_config/migration.sql b/packages/backend/server/migrations/20250325075341_new_app_config/migration.sql new file mode 100644 index 0000000000..35367e4411 --- /dev/null +++ b/packages/backend/server/migrations/20250325075341_new_app_config/migration.sql @@ -0,0 +1,13 @@ +-- CreateTable +CREATE TABLE "app_configs" ( + "id" VARCHAR NOT NULL, + "value" JSONB NOT NULL, + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL, + "last_updated_by" VARCHAR, + + CONSTRAINT "app_configs_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "app_configs" ADD CONSTRAINT "app_configs_last_updated_by_fkey" FOREIGN KEY ("last_updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 232ac8cf73..6f7d0d47e3 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -8,7 +8,7 @@ "run-test": "./scripts/run-test.ts" }, "scripts": { - "build": "tsc", + "build": "tsc -b", "dev": "nodemon ./src/index.ts", "dev:mail": "email dev -d src/mails", "test": "ava --concurrency 1 --serial", @@ -20,6 +20,7 @@ "data-migration": "cross-env NODE_ENV=development r ./src/data/index.ts", "init": "yarn prisma migrate dev && yarn data-migration run", "seed": "r ./src/seed/index.ts", + "genconfig": "r ./scripts/genconfig.ts", "predeploy": "yarn prisma migrate deploy && node --import ./scripts/register.js ./dist/data/index.js run", "postinstall": "prisma generate" }, @@ -139,7 +140,8 @@ "nodemon": "^3.1.7", "react-email": "3.0.7", "sinon": "^19.0.2", - "supertest": "^7.0.0" + "supertest": "^7.0.0", + "why-is-node-running": "^3.2.2" }, "nodemonConfig": { "exec": "node", diff --git a/packages/backend/server/schema.gql b/packages/backend/server/schema.gql new file mode 100644 index 0000000000..af2711c283 --- /dev/null +++ b/packages/backend/server/schema.gql @@ -0,0 +1,1767 @@ +# ------------------------------------------------------ +# THIS FILE WAS AUTOMATICALLY GENERATED (DO NOT MODIFY) +# ------------------------------------------------------ + +input AddContextCategoryInput { + categoryId: String! + contextId: String! + docs: [String!] + type: ContextCategories! +} + +input AddContextDocInput { + contextId: String! + docId: String! +} + +input AddContextFileInput { + blobId: String! + contextId: String! +} + +enum AiJobStatus { + claimed + failed + finished + pending + running +} + +type AlreadyInSpaceDataType { + spaceId: String! +} + +type BlobNotFoundDataType { + blobId: String! + spaceId: String! +} + +enum ChatHistoryOrder { + asc + desc +} + +type ChatMessage { + attachments: [String!] + content: String! + createdAt: DateTime! + id: ID + params: JSON + role: String! +} + +enum ContextCategories { + Collection + Tag +} + +enum ContextEmbedStatus { + failed + finished + processing +} + +type ContextMatchedDocChunk { + chunk: SafeInt! + content: String! + distance: Float + docId: String! +} + +type ContextMatchedFileChunk { + chunk: SafeInt! + content: String! + distance: Float + fileId: String! +} + +type ContextWorkspaceEmbeddingStatus { + embedded: SafeInt! + total: SafeInt! +} + +type Copilot { + audioTranscription(jobId: String): [TranscriptionResultType!]! + + """Get the context list of a session""" + contexts(contextId: String, sessionId: String): [CopilotContext!]! + histories(docId: String, options: QueryChatHistoriesInput): [CopilotHistories!]! + + """Get the quota of the user in the workspace""" + quota: CopilotQuota! + + """Get the session id list in the workspace""" + sessionIds(docId: String, options: QueryChatSessionsInput): [String!]! @deprecated(reason: "Use `sessions` instead") + + """Get the session list in the workspace""" + sessions(docId: String, options: QueryChatSessionsInput): [CopilotSessionType!]! + workspaceId: ID +} + +type CopilotContext { + """list collections in context""" + collections: [CopilotContextCategory!]! + + """list files in context""" + docs: [CopilotContextDoc!]! + + """list files in context""" + files: [CopilotContextFile!]! + id: ID! + + """match file in context""" + matchFiles(content: String!, limit: SafeInt, threshold: Float): [ContextMatchedFileChunk!]! + + """match workspace docs""" + matchWorkspaceDocs(content: String!, limit: SafeInt, threshold: Float): [ContextMatchedDocChunk!]! + + """list tags in context""" + tags: [CopilotContextCategory!]! + workspaceId: String! +} + +type CopilotContextCategory { + createdAt: SafeInt! + docs: [CopilotDocType!]! + id: ID! + type: ContextCategories! +} + +type CopilotContextDoc { + createdAt: SafeInt! + error: String + id: ID! + status: ContextEmbedStatus +} + +type CopilotContextFile { + blobId: String! + chunkSize: SafeInt! + createdAt: SafeInt! + error: String + id: ID! + name: String! + status: ContextEmbedStatus! +} + +type CopilotContextFileNotSupportedDataType { + fileName: String! + message: String! +} + +type CopilotDocNotFoundDataType { + docId: String! +} + +type CopilotDocType { + createdAt: SafeInt! + id: ID! + status: ContextEmbedStatus +} + +type CopilotFailedToMatchContextDataType { + content: String! + contextId: String! + message: String! +} + +type CopilotFailedToModifyContextDataType { + contextId: String! + message: String! +} + +type CopilotHistories { + """An mark identifying which view to use to display the session""" + action: String + createdAt: DateTime! + messages: [ChatMessage!]! + sessionId: String! + + """The number of tokens used in the session""" + tokens: Int! +} + +type CopilotInvalidContextDataType { + contextId: String! +} + +type CopilotMessageNotFoundDataType { + messageId: String! +} + +enum CopilotModels { + DallE3 + Gpt4Omni + Gpt4Omni0806 + Gpt4OmniMini + Gpt4OmniMini0718 + TextEmbedding3Large + TextEmbedding3Small + TextEmbeddingAda002 + TextModerationLatest + TextModerationStable +} + +input CopilotPromptConfigInput { + frequencyPenalty: Float + jsonMode: Boolean + presencePenalty: Float + temperature: Float + topP: Float +} + +type CopilotPromptConfigType { + frequencyPenalty: Float + jsonMode: Boolean + presencePenalty: Float + temperature: Float + topP: Float +} + +input CopilotPromptMessageInput { + content: String! + params: JSON + role: CopilotPromptMessageRole! +} + +enum CopilotPromptMessageRole { + assistant + system + user +} + +type CopilotPromptMessageType { + content: String! + params: JSON + role: CopilotPromptMessageRole! +} + +type CopilotPromptNotFoundDataType { + name: String! +} + +type CopilotPromptType { + action: String + config: CopilotPromptConfigType + messages: [CopilotPromptMessageType!]! + model: String! + name: String! +} + +type CopilotProviderSideErrorDataType { + kind: String! + message: String! + provider: String! +} + +type CopilotQuota { + limit: SafeInt + used: SafeInt! +} + +type CopilotSessionType { + id: ID! + parentSessionId: ID + promptName: String! +} + +input CreateChatMessageInput { + attachments: [String!] + blobs: [Upload!] + content: String + params: JSON + sessionId: String! +} + +input CreateChatSessionInput { + docId: String! + + """The prompt name to use for the session""" + promptName: String! + workspaceId: String! +} + +input CreateCheckoutSessionInput { + args: JSONObject + coupon: String + idempotencyKey: String + plan: SubscriptionPlan = Pro + recurring: SubscriptionRecurring = Yearly + successCallbackLink: String! + variant: SubscriptionVariant +} + +input CreateCopilotPromptInput { + action: String + config: CopilotPromptConfigInput + messages: [CopilotPromptMessageInput!]! + model: CopilotModels! + name: String! +} + +input CreateUserInput { + email: String! + name: String +} + +type CredentialsRequirementType { + password: PasswordLimitsType! +} + +""" +A date-time string at UTC, such as 2019-12-03T09:54:33Z, compliant with the date-time format. +""" +scalar DateTime + +type DeleteAccount { + success: Boolean! +} + +input DeleteSessionInput { + docId: String! + sessionIds: [String!]! + workspaceId: String! +} + +type DocActionDeniedDataType { + action: String! + docId: String! + spaceId: String! +} + +type DocHistoryNotFoundDataType { + docId: String! + spaceId: String! + timestamp: Int! +} + +type DocHistoryType { + editor: EditorType + id: String! + timestamp: DateTime! + workspaceId: String! +} + +"""Doc mode""" +enum DocMode { + edgeless + page +} + +type DocNotFoundDataType { + docId: String! + spaceId: String! +} + +type DocPermissions { + Doc_Copy: Boolean! + Doc_Delete: Boolean! + Doc_Duplicate: Boolean! + Doc_Properties_Read: Boolean! + Doc_Properties_Update: Boolean! + Doc_Publish: Boolean! + Doc_Read: Boolean! + Doc_Restore: Boolean! + Doc_TransferOwner: Boolean! + Doc_Trash: Boolean! + Doc_Update: Boolean! + Doc_Users_Manage: Boolean! + Doc_Users_Read: Boolean! +} + +"""User permission in doc""" +enum DocRole { + Editor + External + Manager + None + Owner + Reader +} + +type DocType { + defaultRole: DocRole! + + """paginated doc granted users list""" + grantedUsersList(pagination: PaginationInput!): PaginatedGrantedDocUserType! + id: String! + mode: PublicDocMode! + permissions: DocPermissions! + public: Boolean! + workspaceId: String! +} + +type DocUpdateBlockedDataType { + docId: String! + spaceId: String! +} + +type EditorType { + avatarUrl: String + name: String! +} + +union ErrorDataUnion = AlreadyInSpaceDataType | BlobNotFoundDataType | CopilotContextFileNotSupportedDataType | CopilotDocNotFoundDataType | CopilotFailedToMatchContextDataType | CopilotFailedToModifyContextDataType | CopilotInvalidContextDataType | CopilotMessageNotFoundDataType | CopilotPromptNotFoundDataType | CopilotProviderSideErrorDataType | DocActionDeniedDataType | DocHistoryNotFoundDataType | DocNotFoundDataType | DocUpdateBlockedDataType | ExpectToGrantDocUserRolesDataType | ExpectToRevokeDocUserRolesDataType | ExpectToUpdateDocUserRoleDataType | GraphqlBadRequestDataType | HttpRequestErrorDataType | InvalidEmailDataType | InvalidHistoryTimestampDataType | InvalidLicenseUpdateParamsDataType | InvalidOauthCallbackCodeDataType | InvalidPasswordLengthDataType | InvalidRuntimeConfigTypeDataType | MemberNotFoundInSpaceDataType | MentionUserDocAccessDeniedDataType | MissingOauthQueryParameterDataType | NotInSpaceDataType | QueryTooLongDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType | SpaceNotFoundDataType | SpaceOwnerNotFoundDataType | SpaceShouldHaveOnlyOneOwnerDataType | SubscriptionAlreadyExistsDataType | SubscriptionNotExistsDataType | SubscriptionPlanNotFoundDataType | UnknownOauthProviderDataType | UnsupportedClientVersionDataType | UnsupportedSubscriptionPlanDataType | ValidationErrorDataType | VersionRejectedDataType | WorkspaceMembersExceedLimitToDowngradeDataType | WorkspacePermissionNotFoundDataType | WrongSignInCredentialsDataType + +enum ErrorNames { + ACCESS_DENIED + ACTION_FORBIDDEN + ACTION_FORBIDDEN_ON_NON_TEAM_WORKSPACE + ALREADY_IN_SPACE + AUTHENTICATION_REQUIRED + BAD_REQUEST + BLOB_NOT_FOUND + BLOB_QUOTA_EXCEEDED + CANNOT_DELETE_ALL_ADMIN_ACCOUNT + CANNOT_DELETE_OWN_ACCOUNT + CANT_UPDATE_ONETIME_PAYMENT_SUBSCRIPTION + CAN_NOT_BATCH_GRANT_DOC_OWNER_PERMISSIONS + CAN_NOT_REVOKE_YOURSELF + CAPTCHA_VERIFICATION_FAILED + COPILOT_ACTION_TAKEN + COPILOT_CONTEXT_FILE_NOT_SUPPORTED + COPILOT_DOCS_NOT_FOUND + COPILOT_DOC_NOT_FOUND + COPILOT_EMBEDDING_UNAVAILABLE + COPILOT_FAILED_TO_CREATE_MESSAGE + COPILOT_FAILED_TO_GENERATE_TEXT + COPILOT_FAILED_TO_MATCH_CONTEXT + COPILOT_FAILED_TO_MODIFY_CONTEXT + COPILOT_INVALID_CONTEXT + COPILOT_MESSAGE_NOT_FOUND + COPILOT_PROMPT_INVALID + COPILOT_PROMPT_NOT_FOUND + COPILOT_PROVIDER_SIDE_ERROR + COPILOT_QUOTA_EXCEEDED + COPILOT_SESSION_DELETED + COPILOT_SESSION_NOT_FOUND + COPILOT_TRANSCRIPTION_JOB_EXISTS + CUSTOMER_PORTAL_CREATE_FAILED + DOC_ACTION_DENIED + DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER + DOC_HISTORY_NOT_FOUND + DOC_IS_NOT_PUBLIC + DOC_NOT_FOUND + DOC_UPDATE_BLOCKED + EARLY_ACCESS_REQUIRED + EMAIL_ALREADY_USED + EMAIL_TOKEN_NOT_FOUND + EMAIL_VERIFICATION_REQUIRED + EXPECT_TO_GRANT_DOC_USER_ROLES + EXPECT_TO_PUBLISH_DOC + EXPECT_TO_REVOKE_DOC_USER_ROLES + EXPECT_TO_REVOKE_PUBLIC_DOC + EXPECT_TO_UPDATE_DOC_USER_ROLE + FAILED_TO_CHECKOUT + FAILED_TO_SAVE_UPDATES + FAILED_TO_UPSERT_SNAPSHOT + GRAPHQL_BAD_REQUEST + HTTP_REQUEST_ERROR + INTERNAL_SERVER_ERROR + INVALID_APP_CONFIG + INVALID_AUTH_STATE + INVALID_CHECKOUT_PARAMETERS + INVALID_EMAIL + INVALID_EMAIL_TOKEN + INVALID_HISTORY_TIMESTAMP + INVALID_LICENSE_SESSION_ID + INVALID_LICENSE_TO_ACTIVATE + INVALID_LICENSE_UPDATE_PARAMS + INVALID_OAUTH_CALLBACK_CODE + INVALID_OAUTH_CALLBACK_STATE + INVALID_PASSWORD_LENGTH + INVALID_RUNTIME_CONFIG_TYPE + INVALID_SUBSCRIPTION_PARAMETERS + LICENSE_NOT_FOUND + LICENSE_REVEALED + LINK_EXPIRED + MAILER_SERVICE_IS_NOT_CONFIGURED + MEMBER_NOT_FOUND_IN_SPACE + MEMBER_QUOTA_EXCEEDED + MENTION_USER_DOC_ACCESS_DENIED + MENTION_USER_ONESELF_DENIED + MISSING_OAUTH_QUERY_PARAMETER + NETWORK_ERROR + NOTIFICATION_NOT_FOUND + NOT_FOUND + NOT_IN_SPACE + NO_COPILOT_PROVIDER_AVAILABLE + OAUTH_ACCOUNT_ALREADY_CONNECTED + OAUTH_STATE_EXPIRED + OWNER_CAN_NOT_LEAVE_WORKSPACE + PASSWORD_REQUIRED + QUERY_TOO_LONG + RUNTIME_CONFIG_NOT_FOUND + SAME_EMAIL_PROVIDED + SAME_SUBSCRIPTION_RECURRING + SIGN_UP_FORBIDDEN + SPACE_ACCESS_DENIED + SPACE_NOT_FOUND + SPACE_OWNER_NOT_FOUND + SPACE_SHOULD_HAVE_ONLY_ONE_OWNER + STORAGE_QUOTA_EXCEEDED + SUBSCRIPTION_ALREADY_EXISTS + SUBSCRIPTION_EXPIRED + SUBSCRIPTION_HAS_BEEN_CANCELED + SUBSCRIPTION_HAS_NOT_BEEN_CANCELED + SUBSCRIPTION_NOT_EXISTS + SUBSCRIPTION_PLAN_NOT_FOUND + TOO_MANY_REQUEST + UNKNOWN_OAUTH_PROVIDER + UNSPLASH_IS_NOT_CONFIGURED + UNSUPPORTED_CLIENT_VERSION + UNSUPPORTED_SUBSCRIPTION_PLAN + USER_AVATAR_NOT_FOUND + USER_NOT_FOUND + VALIDATION_ERROR + VERSION_REJECTED + WORKSPACE_ID_REQUIRED_FOR_TEAM_SUBSCRIPTION + WORKSPACE_ID_REQUIRED_TO_UPDATE_TEAM_SUBSCRIPTION + WORKSPACE_LICENSE_ALREADY_EXISTS + WORKSPACE_MEMBERS_EXCEED_LIMIT_TO_DOWNGRADE + WORKSPACE_PERMISSION_NOT_FOUND + WRONG_SIGN_IN_CREDENTIALS + WRONG_SIGN_IN_METHOD +} + +type ExpectToGrantDocUserRolesDataType { + docId: String! + spaceId: String! +} + +type ExpectToRevokeDocUserRolesDataType { + docId: String! + spaceId: String! +} + +type ExpectToUpdateDocUserRoleDataType { + docId: String! + spaceId: String! +} + +enum FeatureType { + AIEarlyAccess + Admin + EarlyAccess + FreePlan + LifetimeProPlan + ProPlan + TeamPlan + UnlimitedCopilot + UnlimitedWorkspace +} + +input ForkChatSessionInput { + docId: String! + + """ + Identify a message in the array and keep it with all previous messages into a forked session. + """ + latestMessageId: String! + sessionId: String! + workspaceId: String! +} + +input GrantDocUserRolesInput { + docId: String! + role: DocRole! + userIds: [String!]! + workspaceId: String! +} + +type GrantedDocUserType { + role: DocRole! + user: WorkspaceUserType! +} + +type GrantedDocUserTypeEdge { + cursor: String! + node: GrantedDocUserType! +} + +type GraphqlBadRequestDataType { + code: String! + message: String! +} + +type HttpRequestErrorDataType { + message: String! +} + +input ImportUsersInput { + users: [CreateUserInput!]! +} + +type InvalidEmailDataType { + email: String! +} + +type InvalidHistoryTimestampDataType { + timestamp: String! +} + +type InvalidLicenseUpdateParamsDataType { + reason: String! +} + +type InvalidOauthCallbackCodeDataType { + body: String! + status: Int! +} + +type InvalidPasswordLengthDataType { + max: Int! + min: Int! +} + +type InvalidRuntimeConfigTypeDataType { + get: String! + key: String! + want: String! +} + +type InvitationAcceptedNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + inviteId: ID! + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type InvitationBlockedNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + inviteId: ID! + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type InvitationNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + inviteId: ID! + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type InvitationReviewApprovedNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + inviteId: ID! + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type InvitationReviewDeclinedNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type InvitationReviewRequestNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + inviteId: ID! + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type InvitationType { + """Invitee information""" + invitee: WorkspaceUserType! + + """Invitation status in workspace""" + status: WorkspaceMemberStatus + + """User information""" + user: WorkspaceUserType! + + """Workspace information""" + workspace: InvitationWorkspaceType! +} + +type InvitationWorkspaceType { + """Base64 encoded avatar""" + avatar: String! + id: ID! + + """Workspace name""" + name: String! +} + +type InviteLink { + """Invite link expire time""" + expireTime: DateTime! + + """Invite link""" + link: String! +} + +type InviteResult { + email: String! + + """Invite id, null if invite record create failed""" + inviteId: String + + """Invite email sent success""" + sentSuccess: Boolean! +} + +type InviteUserType { + """User accepted""" + accepted: Boolean! @deprecated(reason: "Use `status` instead") + + """User avatar url""" + avatarUrl: String + + """User email verified""" + createdAt: DateTime @deprecated(reason: "useless") + + """User is disabled""" + disabled: Boolean + + """User email""" + email: String + + """User email verified""" + emailVerified: Boolean + + """User password has been set""" + hasPassword: Boolean + id: ID! + + """Invite id""" + inviteId: String! + + """User name""" + name: String + + """User permission in workspace""" + permission: Permission! @deprecated(reason: "Use role instead") + + """User role in workspace""" + role: Permission! + + """Member invite status in workspace""" + status: WorkspaceMemberStatus! +} + +enum InvoiceStatus { + Draft + Open + Paid + Uncollectible + Void +} + +type InvoiceType { + amount: Int! + createdAt: DateTime! + currency: String! + id: String @deprecated(reason: "removed") + lastPaymentError: String + link: String + plan: SubscriptionPlan @deprecated(reason: "removed") + reason: String! + recurring: SubscriptionRecurring @deprecated(reason: "removed") + status: InvoiceStatus! + updatedAt: DateTime! +} + +""" +The `JSON` scalar type represents JSON values as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). +""" +scalar JSON @specifiedBy(url: "http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf") + +""" +The `JSONObject` scalar type represents JSON objects as specified by [ECMA-404](http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf). +""" +scalar JSONObject @specifiedBy(url: "http://www.ecma-international.org/publications/files/ECMA-ST/ECMA-404.pdf") + +type License { + expiredAt: DateTime + installedAt: DateTime! + quantity: Int! + recurring: SubscriptionRecurring! + validatedAt: DateTime! +} + +type LimitedUserType { + """User email""" + email: String! + + """User password has been set""" + hasPassword: Boolean +} + +input ListUserInput { + first: Int = 20 + skip: Int = 0 +} + +type ListedBlob { + createdAt: String! + key: String! + mime: String! + size: Int! +} + +input ManageUserInput { + """User email""" + email: String + + """User name""" + name: String +} + +type MemberNotFoundInSpaceDataType { + spaceId: String! +} + +input MentionDocInput { + """The block id in the doc""" + blockId: String + + """The element id in the doc""" + elementId: String + id: String! + mode: DocMode! + title: String! +} + +type MentionDocType { + blockId: String + elementId: String + id: String! + mode: DocMode! + title: String! +} + +input MentionInput { + doc: MentionDocInput! + userId: String! + workspaceId: String! +} + +type MentionNotificationBodyType { + """ + The user who created the notification, maybe null when user is deleted or sent by system + """ + createdByUser: PublicUserType + doc: MentionDocType! + + """The type of the notification""" + type: NotificationType! + workspace: NotificationWorkspaceType +} + +type MentionUserDocAccessDeniedDataType { + docId: String! +} + +type MissingOauthQueryParameterDataType { + name: String! +} + +type Mutation { + acceptInviteById(inviteId: String!, sendAcceptMail: Boolean @deprecated(reason: "never used"), workspaceId: String!): Boolean! + activateLicense(license: String!, workspaceId: String!): License! + + """add a category to context""" + addContextCategory(options: AddContextCategoryInput!): CopilotContextCategory! + + """add a doc to context""" + addContextDoc(options: AddContextDocInput!): CopilotContextDoc! + + """add a file to context""" + addContextFile(content: Upload!, options: AddContextFileInput!): CopilotContextFile! + addWorkspaceFeature(feature: FeatureType!, workspaceId: String!): Boolean! + approveMember(userId: String!, workspaceId: String!): Boolean! + + """Ban an user""" + banUser(id: String!): UserType! + cancelSubscription(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, workspaceId: String): SubscriptionType! + changeEmail(email: String!, token: String!): UserType! + changePassword(newPassword: String!, token: String!, userId: String): Boolean! + claimAudioTranscription(jobId: String!): TranscriptionResultType + + """Cleanup sessions""" + cleanupCopilotSession(options: DeleteSessionInput!): [String!]! + + """Create change password url""" + createChangePasswordUrl(callbackUrl: String!, userId: String!): String! + + """Create a subscription checkout link of stripe""" + createCheckoutSession(input: CreateCheckoutSessionInput!): String! + + """Create a context session""" + createCopilotContext(sessionId: String!, workspaceId: String!): String! + + """Create a chat message""" + createCopilotMessage(options: CreateChatMessageInput!): String! + + """Create a copilot prompt""" + createCopilotPrompt(input: CreateCopilotPromptInput!): CopilotPromptType! + + """Create a chat session""" + createCopilotSession(options: CreateChatSessionInput!): String! + + """Create a stripe customer portal to manage payment methods""" + createCustomerPortal: String! + createInviteLink(expireTime: WorkspaceInviteLinkExpireTime!, workspaceId: String!): InviteLink! + createSelfhostWorkspaceCustomerPortal(workspaceId: String!): String! + + """Create a new user""" + createUser(input: CreateUserInput!): UserType! + + """Create a new workspace""" + createWorkspace(init: Upload): WorkspaceType! + deactivateLicense(workspaceId: String!): Boolean! + deleteAccount: DeleteAccount! + deleteBlob(hash: String @deprecated(reason: "use parameter [key]"), key: String, permanently: Boolean! = false, workspaceId: String!): Boolean! + + """Delete a user account""" + deleteUser(id: String!): DeleteAccount! + deleteWorkspace(id: String!): Boolean! + + """Reenable an banned user""" + enableUser(id: String!): UserType! + + """Create a chat session""" + forkCopilotSession(options: ForkChatSessionInput!): String! + generateLicenseKey(sessionId: String!): String! + grantDocUserRoles(input: GrantDocUserRolesInput!): Boolean! + grantMember(permission: Permission!, userId: String!, workspaceId: String!): Boolean! + + """import users""" + importUsers(input: ImportUsersInput!): [UserImportResultType!]! + invite(email: String!, permission: Permission @deprecated(reason: "never used"), sendInviteMail: Boolean @deprecated(reason: "never used"), workspaceId: String!): String! + inviteBatch(emails: [String!]!, sendInviteMail: Boolean @deprecated(reason: "never used"), workspaceId: String!): [InviteResult!]! + leaveWorkspace(sendLeaveMail: Boolean, workspaceId: String!, workspaceName: String @deprecated(reason: "no longer used")): Boolean! + + """mention user in a doc""" + mentionUser(input: MentionInput!): ID! + publishDoc(docId: String!, mode: PublicDocMode = Page, workspaceId: String!): DocType! + publishPage(mode: PublicDocMode = Page, pageId: String!, workspaceId: String!): DocType! @deprecated(reason: "use publishDoc instead") + + """queue workspace doc embedding""" + queueWorkspaceEmbedding(docId: [String!]!, workspaceId: String!): Boolean! + + """mark notification as read""" + readNotification(id: String!): Boolean! + recoverDoc(guid: String!, timestamp: DateTime!, workspaceId: String!): DateTime! + releaseDeletedBlobs(workspaceId: String!): Boolean! + + """Remove user avatar""" + removeAvatar: RemoveAvatar! + + """remove a category from context""" + removeContextCategory(options: RemoveContextCategoryInput!): Boolean! + + """remove a doc from context""" + removeContextDoc(options: RemoveContextDocInput!): Boolean! + + """remove a file from context""" + removeContextFile(options: RemoveContextFileInput!): Boolean! + removeWorkspaceFeature(feature: FeatureType!, workspaceId: String!): Boolean! + resumeSubscription(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, workspaceId: String): SubscriptionType! + revoke(userId: String!, workspaceId: String!): Boolean! + revokeDocUserRoles(input: RevokeDocUserRoleInput!): Boolean! + revokeInviteLink(workspaceId: String!): Boolean! + revokePublicDoc(docId: String!, workspaceId: String!): DocType! + revokePublicPage(docId: String!, workspaceId: String!): DocType! @deprecated(reason: "use revokePublicDoc instead") + sendChangeEmail(callbackUrl: String!, email: String): Boolean! + sendChangePasswordEmail(callbackUrl: String!, email: String @deprecated(reason: "fetched from signed in user")): Boolean! + sendSetPasswordEmail(callbackUrl: String!, email: String @deprecated(reason: "fetched from signed in user")): Boolean! + sendVerifyChangeEmail(callbackUrl: String!, email: String!, token: String!): Boolean! + sendVerifyEmail(callbackUrl: String!): Boolean! + setBlob(blob: Upload!, workspaceId: String!): String! + submitAudioTranscription(blob: Upload!, blobId: String!, workspaceId: String!): TranscriptionResultType + + """update app configuration""" + updateAppConfig(updates: [UpdateAppConfigInput!]!): JSONObject! + + """Update a copilot prompt""" + updateCopilotPrompt(messages: [CopilotPromptMessageInput!]!, name: String!): CopilotPromptType! + + """Update a chat session""" + updateCopilotSession(options: UpdateChatSessionInput!): String! + updateDocDefaultRole(input: UpdateDocDefaultRoleInput!): Boolean! + updateDocUserRole(input: UpdateDocUserRoleInput!): Boolean! + updateProfile(input: UpdateUserInput!): UserType! + + """Update user settings""" + updateSettings(input: UpdateUserSettingsInput!): Boolean! + updateSubscriptionRecurring(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, recurring: SubscriptionRecurring!, workspaceId: String): SubscriptionType! + + """Update an user""" + updateUser(id: String!, input: ManageUserInput!): UserType! + + """update user enabled feature""" + updateUserFeatures(features: [FeatureType!]!, id: String!): [FeatureType!]! + + """Update workspace""" + updateWorkspace(input: UpdateWorkspaceInput!): WorkspaceType! + + """Upload user avatar""" + uploadAvatar(avatar: Upload!): UserType! + verifyEmail(token: String!): Boolean! +} + +type NotInSpaceDataType { + spaceId: String! +} + +"""Notification level""" +enum NotificationLevel { + Default + High + Low + Min + None +} + +type NotificationObjectType { + """Just a placeholder to export UnionNotificationBodyType, don't use it""" + _placeholderForUnionNotificationBodyType: UnionNotificationBodyType! + + """ + The body of the notification, different types have different fields, see UnionNotificationBodyType + """ + body: JSONObject! + + """The created at time of the notification""" + createdAt: DateTime! + id: ID! + + """The level of the notification""" + level: NotificationLevel! + + """Whether the notification has been read""" + read: Boolean! + + """The type of the notification""" + type: NotificationType! + + """The updated at time of the notification""" + updatedAt: DateTime! +} + +type NotificationObjectTypeEdge { + cursor: String! + node: NotificationObjectType! +} + +"""Notification type""" +enum NotificationType { + Invitation + InvitationAccepted + InvitationBlocked + InvitationRejected + InvitationReviewApproved + InvitationReviewDeclined + InvitationReviewRequest + Mention +} + +type NotificationWorkspaceType { + """Workspace avatar url""" + avatarUrl: String + id: ID! + + """Workspace name""" + name: String! +} + +enum OAuthProviderType { + GitHub + Google + OIDC +} + +type PageInfo { + endCursor: String + hasNextPage: Boolean! + hasPreviousPage: Boolean! + startCursor: String +} + +type PaginatedGrantedDocUserType { + edges: [GrantedDocUserTypeEdge!]! + pageInfo: PageInfo! + totalCount: Int! +} + +type PaginatedNotificationObjectType { + edges: [NotificationObjectTypeEdge!]! + pageInfo: PageInfo! + totalCount: Int! +} + +input PaginationInput { + """returns the elements in the list that come after the specified cursor.""" + after: String + + """returns the first n elements from the list.""" + first: Int = 10 + + """ignore the first n elements from the list.""" + offset: Int = 0 +} + +type PasswordLimitsType { + maxLength: Int! + minLength: Int! +} + +"""User permission in workspace""" +enum Permission { + Admin + Collaborator + External + Owner +} + +"""The mode which the public doc default in""" +enum PublicDocMode { + Edgeless + Page +} + +type PublicUserType { + avatarUrl: String + id: String! + name: String! +} + +type Query { + """get the whole app configuration""" + appConfig: JSONObject! + collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.quotaUsage` instead") + + """Get current user""" + currentUser: UserType + error(name: ErrorNames!): ErrorDataUnion! + + """send workspace invitation""" + getInviteInfo(inviteId: String!): InvitationType! + + """Get is admin of workspace""" + isAdmin(workspaceId: String!): Boolean! @deprecated(reason: "use WorkspaceType[role] instead") + + """Get is owner of workspace""" + isOwner(workspaceId: String!): Boolean! @deprecated(reason: "use WorkspaceType[role] instead") + + """List all copilot prompts""" + listCopilotPrompts: [CopilotPromptType!]! + prices: [SubscriptionPrice!]! + + """Get public user by id""" + publicUserById(id: String!): PublicUserType + + """query workspace embedding status""" + queryWorkspaceEmbeddingStatus(workspaceId: String!): ContextWorkspaceEmbeddingStatus! + + """server config""" + serverConfig: ServerConfigType! + + """Get user by email""" + user(email: String!): UserOrLimitedUser + + """Get user by email for admin""" + userByEmail(email: String!): UserType + + """Get user by id""" + userById(id: String!): UserType! + + """List registered users""" + users(filter: ListUserInput!): [UserType!]! + + """Get users count""" + usersCount: Int! + + """Get workspace by id""" + workspace(id: String!): WorkspaceType! + + """Get workspace role permissions""" + workspaceRolePermissions(id: String!): WorkspaceRolePermissions! @deprecated(reason: "use WorkspaceType[permissions] instead") + + """Get all accessible workspaces for current user""" + workspaces: [WorkspaceType!]! +} + +input QueryChatHistoriesInput { + action: Boolean + fork: Boolean + limit: Int + messageOrder: ChatHistoryOrder + sessionId: String + sessionOrder: ChatHistoryOrder + skip: Int + withPrompt: Boolean +} + +input QueryChatSessionsInput { + action: Boolean +} + +type QueryTooLongDataType { + max: Int! +} + +type ReleaseVersionType { + changelog: String! + publishedAt: DateTime! + url: String! + version: String! +} + +type RemoveAvatar { + success: Boolean! +} + +input RemoveContextCategoryInput { + categoryId: String! + contextId: String! + type: ContextCategories! +} + +input RemoveContextDocInput { + contextId: String! + docId: String! +} + +input RemoveContextFileInput { + contextId: String! + fileId: String! +} + +input RevokeDocUserRoleInput { + docId: String! + userId: String! + workspaceId: String! +} + +type RuntimeConfigNotFoundDataType { + key: String! +} + +""" +The `SafeInt` scalar type represents non-fractional signed whole numeric values that are considered safe as defined by the ECMAScript specification. +""" +scalar SafeInt @specifiedBy(url: "https://www.ecma-international.org/ecma-262/#sec-number.issafeinteger") + +type SameSubscriptionRecurringDataType { + recurring: String! +} + +type ServerConfigType { + """fetch latest available upgradable release of server""" + availableUpgrade: ReleaseVersionType + + """Features for user that can be configured""" + availableUserFeatures: [FeatureType!]! + + """server base url""" + baseUrl: String! + + """credentials requirement""" + credentialsRequirement: CredentialsRequirementType! + + """enabled server features""" + features: [ServerFeature!]! + + """whether server has been initialized""" + initialized: Boolean! + + """server identical name could be shown as badge on user interface""" + name: String! + oauthProviders: [OAuthProviderType!]! + + """server type""" + type: ServerDeploymentType! + + """server version""" + version: String! +} + +enum ServerDeploymentType { + Affine + Selfhosted +} + +enum ServerFeature { + Captcha + Copilot + OAuth + Payment +} + +type SpaceAccessDeniedDataType { + spaceId: String! +} + +type SpaceNotFoundDataType { + spaceId: String! +} + +type SpaceOwnerNotFoundDataType { + spaceId: String! +} + +type SpaceShouldHaveOnlyOneOwnerDataType { + spaceId: String! +} + +type SubscriptionAlreadyExistsDataType { + plan: String! +} + +type SubscriptionNotExistsDataType { + plan: String! +} + +enum SubscriptionPlan { + AI + Enterprise + Free + Pro + SelfHosted + SelfHostedTeam + Team +} + +type SubscriptionPlanNotFoundDataType { + plan: String! + recurring: String! +} + +type SubscriptionPrice { + amount: Int + currency: String! + lifetimeAmount: Int + plan: SubscriptionPlan! + type: String! + yearlyAmount: Int +} + +enum SubscriptionRecurring { + Lifetime + Monthly + Yearly +} + +enum SubscriptionStatus { + Active + Canceled + Incomplete + IncompleteExpired + PastDue + Paused + Trialing + Unpaid +} + +type SubscriptionType { + canceledAt: DateTime + createdAt: DateTime! + end: DateTime + id: String @deprecated(reason: "removed") + nextBillAt: DateTime + + """ + The 'Free' plan just exists to be a placeholder and for the type convenience of frontend. + There won't actually be a subscription with plan 'Free' + """ + plan: SubscriptionPlan! + recurring: SubscriptionRecurring! + start: DateTime! + status: SubscriptionStatus! + trialEnd: DateTime + trialStart: DateTime + updatedAt: DateTime! + variant: SubscriptionVariant +} + +enum SubscriptionVariant { + EA + Onetime +} + +type TranscriptionItemType { + end: String! + speaker: String! + start: String! + transcription: String! +} + +type TranscriptionResultType { + id: ID! + status: AiJobStatus! + summary: String + transcription: [TranscriptionItemType!] +} + +union UnionNotificationBodyType = InvitationAcceptedNotificationBodyType | InvitationBlockedNotificationBodyType | InvitationNotificationBodyType | InvitationReviewApprovedNotificationBodyType | InvitationReviewDeclinedNotificationBodyType | InvitationReviewRequestNotificationBodyType | MentionNotificationBodyType + +type UnknownOauthProviderDataType { + name: String! +} + +type UnsupportedClientVersionDataType { + clientVersion: String! + requiredVersion: String! +} + +type UnsupportedSubscriptionPlanDataType { + plan: String! +} + +input UpdateAppConfigInput { + key: String! + module: String! + value: JSON! +} + +input UpdateChatSessionInput { + """The prompt name to use for the session""" + promptName: String! + sessionId: String! +} + +input UpdateDocDefaultRoleInput { + docId: String! + role: DocRole! + workspaceId: String! +} + +input UpdateDocUserRoleInput { + docId: String! + role: DocRole! + userId: String! + workspaceId: String! +} + +input UpdateUserInput { + """User name""" + name: String +} + +input UpdateUserSettingsInput { + """Receive invitation email""" + receiveInvitationEmail: Boolean + + """Receive mention email""" + receiveMentionEmail: Boolean +} + +input UpdateWorkspaceInput { + """Enable AI""" + enableAi: Boolean + + """Enable url previous when sharing""" + enableUrlPreview: Boolean + id: ID! + + """is Public workspace""" + public: Boolean +} + +"""The `Upload` scalar type represents a file upload.""" +scalar Upload + +type UserImportFailedType { + email: String! + error: String! +} + +union UserImportResultType = UserImportFailedType | UserType + +union UserOrLimitedUser = LimitedUserType | UserType + +type UserQuotaHumanReadableType { + blobLimit: String! + copilotActionLimit: String! + historyPeriod: String! + memberLimit: String! + name: String! + storageQuota: String! + usedStorageQuota: String! +} + +type UserQuotaType { + blobLimit: SafeInt! + copilotActionLimit: Int + historyPeriod: SafeInt! + humanReadable: UserQuotaHumanReadableType! + memberLimit: Int! + name: String! + storageQuota: SafeInt! + usedStorageQuota: SafeInt! +} + +type UserQuotaUsageType { + storageQuota: SafeInt! @deprecated(reason: "use `UserQuotaType['usedStorageQuota']` instead") +} + +type UserSettingsType { + """Receive invitation email""" + receiveInvitationEmail: Boolean! + + """Receive mention email""" + receiveMentionEmail: Boolean! +} + +type UserType { + """User avatar url""" + avatarUrl: String + copilot(workspaceId: String): Copilot! + + """User email verified""" + createdAt: DateTime @deprecated(reason: "useless") + + """User is disabled""" + disabled: Boolean! + + """User email""" + email: String! + + """User email verified""" + emailVerified: Boolean! + + """Enabled features of a user""" + features: [FeatureType!]! + + """User password has been set""" + hasPassword: Boolean + id: ID! + + """Get user invoice count""" + invoiceCount: Int! + invoices(skip: Int, take: Int = 8): [InvoiceType!]! + + """User name""" + name: String! + + """Get user notification count""" + notificationCount: Int! + + """Get current user notifications""" + notifications(pagination: PaginationInput!): PaginatedNotificationObjectType! + quota: UserQuotaType! + quotaUsage: UserQuotaUsageType! + + """Get user settings""" + settings: UserSettingsType! + subscriptions: [SubscriptionType!]! + token: tokenType! @deprecated(reason: "use [/api/auth/sign-in?native=true] instead") +} + +type ValidationErrorDataType { + errors: String! +} + +type VersionRejectedDataType { + serverVersion: String! + version: String! +} + +type WorkspaceBlobSizes { + size: SafeInt! +} + +"""Workspace invite link expire time""" +enum WorkspaceInviteLinkExpireTime { + OneDay + OneMonth + OneWeek + ThreeDays +} + +"""Member invite status in workspace""" +enum WorkspaceMemberStatus { + Accepted + NeedMoreSeat + NeedMoreSeatAndReview + Pending + UnderReview +} + +type WorkspaceMembersExceedLimitToDowngradeDataType { + limit: Int! +} + +type WorkspacePageMeta { + createdAt: DateTime! + createdBy: EditorType + updatedAt: DateTime! + updatedBy: EditorType +} + +type WorkspacePermissionNotFoundDataType { + spaceId: String! +} + +type WorkspacePermissions { + Workspace_Administrators_Manage: Boolean! + Workspace_Blobs_List: Boolean! + Workspace_Blobs_Read: Boolean! + Workspace_Blobs_Write: Boolean! + Workspace_Copilot: Boolean! + Workspace_CreateDoc: Boolean! + Workspace_Delete: Boolean! + Workspace_Organize_Read: Boolean! + Workspace_Payment_Manage: Boolean! + Workspace_Properties_Create: Boolean! + Workspace_Properties_Delete: Boolean! + Workspace_Properties_Read: Boolean! + Workspace_Properties_Update: Boolean! + Workspace_Read: Boolean! + Workspace_Settings_Read: Boolean! + Workspace_Settings_Update: Boolean! + Workspace_Sync: Boolean! + Workspace_TransferOwner: Boolean! + Workspace_Users_Manage: Boolean! + Workspace_Users_Read: Boolean! +} + +type WorkspaceQuotaHumanReadableType { + blobLimit: String! + historyPeriod: String! + memberCount: String! + memberLimit: String! + name: String! + storageQuota: String! + storageQuotaUsed: String! +} + +type WorkspaceQuotaType { + blobLimit: SafeInt! + historyPeriod: SafeInt! + humanReadable: WorkspaceQuotaHumanReadableType! + memberCount: Int! + memberLimit: Int! + name: String! + storageQuota: SafeInt! + usedSize: SafeInt! @deprecated(reason: "use `usedStorageQuota` instead") + usedStorageQuota: SafeInt! +} + +type WorkspaceRolePermissions { + permissions: WorkspacePermissions! + role: Permission! +} + +type WorkspaceType { + """List blobs of workspace""" + blobs: [ListedBlob!]! + + """Blobs size of workspace""" + blobsSize: Int! + + """Workspace created date""" + createdAt: DateTime! + + """Get get with given id""" + doc(docId: String!): DocType! + + """Enable AI""" + enableAi: Boolean! + + """Enable url previous when sharing""" + enableUrlPreview: Boolean! + histories(before: DateTime, guid: String!, take: Int): [DocHistoryType!]! + id: ID! + + """is current workspace initialized""" + initialized: Boolean! + + """invite link for workspace""" + inviteLink: InviteLink + + """Get user invoice count""" + invoiceCount: Int! + invoices(skip: Int, take: Int = 8): [InvoiceType!]! + + """The selfhost license of the workspace""" + license: License + + """member count of workspace""" + memberCount: Int! + + """Members of workspace""" + members(query: String, skip: Int, take: Int): [InviteUserType!]! + + """Owner of workspace""" + owner: UserType! + + """Cloud page metadata of workspace""" + pageMeta(pageId: String!): WorkspacePageMeta! + + """map of action permissions""" + permissions: WorkspacePermissions! + + """is Public workspace""" + public: Boolean! + + """Get public docs of a workspace""" + publicDocs: [DocType!]! + + """Get public page of a workspace by page id.""" + publicPage(pageId: String!): DocType @deprecated(reason: "use [WorkspaceType.doc] instead") + publicPages: [DocType!]! @deprecated(reason: "use [WorkspaceType.publicDocs] instead") + + """quota of workspace""" + quota: WorkspaceQuotaType! + + """Role of current signed in user in workspace""" + role: Permission! + + """The team subscription of the workspace, if exists.""" + subscription: SubscriptionType + + """if workspace is team workspace""" + team: Boolean! +} + +type WorkspaceUserType { + avatarUrl: String + email: String! + id: String! + name: String! +} + +type WrongSignInCredentialsDataType { + email: String! +} + +type tokenType { + refresh: String! + sessionToken: String + token: String! +} \ No newline at end of file diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prisma index 49f4eb5305..d91469652c 100644 --- a/packages/backend/server/schema.prisma +++ b/packages/backend/server/schema.prisma @@ -24,23 +24,25 @@ model User { registered Boolean @default(true) disabled Boolean @default(false) - features UserFeature[] - userStripeCustomer UserStripeCustomer? - workspacePermissions WorkspaceUserRole[] - docPermissions WorkspaceDocUserRole[] - connectedAccounts ConnectedAccount[] - sessions UserSession[] - aiSessions AiSession[] - updatedRuntimeConfigs RuntimeConfig[] - userSnapshots UserSnapshot[] - createdSnapshot Snapshot[] @relation("createdSnapshot") - updatedSnapshot Snapshot[] @relation("updatedSnapshot") - createdUpdate Update[] @relation("createdUpdate") - createdHistory SnapshotHistory[] @relation("createdHistory") - createdAiJobs AiJobs[] @relation("createdAiJobs") + features UserFeature[] + userStripeCustomer UserStripeCustomer? + workspacePermissions WorkspaceUserRole[] + docPermissions WorkspaceDocUserRole[] + connectedAccounts ConnectedAccount[] + sessions UserSession[] + aiSessions AiSession[] + /// @deprecated + deprecatedAppRuntimeSettings DeprecatedAppRuntimeSettings[] + appConfigs AppConfig[] + userSnapshots UserSnapshot[] + createdSnapshot Snapshot[] @relation("createdSnapshot") + updatedSnapshot Snapshot[] @relation("updatedSnapshot") + createdUpdate Update[] @relation("createdUpdate") + createdHistory SnapshotHistory[] @relation("createdHistory") + createdAiJobs AiJobs[] @relation("createdAiJobs") // receive notifications - notifications Notification[] @relation("user_notifications") - settings UserSettings? + notifications Notification[] @relation("user_notifications") + settings UserSettings? @@index([email]) @@map("users") @@ -438,12 +440,12 @@ model AiContext { } model AiContextEmbedding { - id String @id @default(uuid()) @db.VarChar - contextId String @map("context_id") @db.VarChar - fileId String @map("file_id") @db.VarChar + id String @id @default(uuid()) @db.VarChar + contextId String @map("context_id") @db.VarChar + fileId String @map("file_id") @db.VarChar // a file can be divided into multiple chunks and embedded separately. - chunk Int @db.Integer - content String @db.VarChar + chunk Int @db.Integer + content String @db.VarChar embedding Unsupported("vector(1024)") createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) @@ -457,11 +459,11 @@ model AiContextEmbedding { } model AiWorkspaceEmbedding { - workspaceId String @map("workspace_id") @db.VarChar - docId String @map("doc_id") @db.VarChar + workspaceId String @map("workspace_id") @db.VarChar + docId String @map("doc_id") @db.VarChar // a doc can be divided into multiple chunks and embedded separately. - chunk Int @db.Integer - content String @db.VarChar + chunk Int @db.Integer + content String @db.VarChar embedding Unsupported("vector(1024)") createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) @@ -527,7 +529,8 @@ enum RuntimeConfigType { Array } -model RuntimeConfig { +/// @deprecated use AppConfig instead +model DeprecatedAppRuntimeSettings { id String @id @db.VarChar type RuntimeConfigType module String @db.VarChar @@ -544,6 +547,18 @@ model RuntimeConfig { @@map("app_runtime_settings") } +model AppConfig { + id String @id @db.VarChar + value Json @db.JsonB + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz(3) + lastUpdatedBy String? @map("last_updated_by") @db.VarChar + + lastUpdatedByUser User? @relation(fields: [lastUpdatedBy], references: [id], onDelete: SetNull) + + @@map("app_configs") +} + model DeprecatedUserSubscription { id Int @id @default(autoincrement()) @db.Integer userId String @map("user_id") @db.VarChar diff --git a/packages/backend/server/scripts/genconfig.ts b/packages/backend/server/scripts/genconfig.ts new file mode 100644 index 0000000000..5e933ada24 --- /dev/null +++ b/packages/backend/server/scripts/genconfig.ts @@ -0,0 +1,101 @@ +/* eslint-disable */ +import '../src/prelude'; +import '../src/app.module'; + +import fs from 'node:fs'; +import { ProjectRoot } from '@affine-tools/utils/path'; +import { Package } from '@affine-tools/utils/workspace'; +import { getDescriptors, ConfigDescriptor } from '../src/base/config/register'; +import { pick } from 'lodash-es'; + +interface PropertySchema { + description: string; + type?: 'array' | 'boolean' | 'integer' | 'number' | 'object' | 'string'; + default?: any; +} + +function convertDescriptorToSchemaProperty(descriptor: ConfigDescriptor) { + const property: PropertySchema = { + ...descriptor.schema, + description: + descriptor.schema.description + + `\n@default ${JSON.stringify(descriptor.default)}` + + (descriptor.env ? `\n@environment \`${descriptor.env[0]}\`` : '') + + (descriptor.link ? `\n@link ${descriptor.link}` : ''), + default: descriptor.default, + }; + + return property; +} + +function generateJsonSchema(outputPath: string) { + const schema = { + $schema: 'http://json-schema.org/draft-07/schema#', + title: 'AFFiNE Application Configuration', + type: 'object', + properties: {}, + }; + + getDescriptors().forEach(({ module, descriptors }) => { + schema.properties[module] = { + type: 'object', + description: `Configuration for ${module} module`, + properties: {}, + }; + + descriptors.forEach(({ key, descriptor }) => { + schema.properties[module].properties[key] = + convertDescriptorToSchemaProperty(descriptor); + }); + }); + + fs.writeFileSync(outputPath, JSON.stringify(schema, null, 2)); + + console.log(`Config schema generated at: ${outputPath}`); +} + +function generateAdminConfigJson(outputPath: string) { + const config = {}; + getDescriptors().forEach(({ module, descriptors }) => { + const modulizedConfig = {}; + config[module] = modulizedConfig; + descriptors.forEach(({ key, descriptor }) => { + let type: string; + switch (descriptor.schema?.type) { + case 'number': + type = 'Number'; + break; + case 'boolean': + type = 'Boolean'; + break; + case 'array': + type = 'Array'; + break; + case 'object': + type = 'Object'; + break; + default: + type = 'String'; + } + + modulizedConfig[key] = { + type, + desc: descriptor.desc, + link: descriptor.link, + env: descriptor.env?.[0], + }; + }); + }); + fs.writeFileSync(outputPath, JSON.stringify(config, null, 2)); +} + +function main() { + generateJsonSchema( + ProjectRoot.join('.docker', 'selfhost', 'schema.json').toString() + ); + generateAdminConfigJson( + new Package('@affine/admin').join('src/config.json').toString() + ); +} + +main(); diff --git a/packages/backend/server/scripts/self-host-predeploy.js b/packages/backend/server/scripts/self-host-predeploy.js index ba5c43105f..4132642455 100644 --- a/packages/backend/server/scripts/self-host-predeploy.js +++ b/packages/backend/server/scripts/self-host-predeploy.js @@ -1,17 +1,10 @@ import { execSync } from 'node:child_process'; import { generateKeyPairSync } from 'node:crypto'; import fs from 'node:fs'; +import { homedir } from 'node:os'; import path from 'node:path'; -const SELF_HOST_CONFIG_DIR = '/root/.affine/config'; - -function generateConfigFile() { - const content = fs.readFileSync('./dist/config/affine.js', 'utf-8'); - return content.replace( - /(^\/\/#.*$)|(^\/\/\s+TODO.*$)|("use\sstrict";?)|(^.*lint-disable.*$)/gm, - '' - ); -} +const SELF_HOST_CONFIG_DIR = `${homedir()}/.affine/config`; function generatePrivateKey() { const key = generateKeyPairSync('ec', { @@ -31,15 +24,12 @@ function generatePrivateKey() { /** * @type {Array<{ to: string; generator: () => string }>} */ -const configFiles = [ - { to: 'affine.js', generator: generateConfigFile }, - { to: 'private.key', generator: generatePrivateKey }, -]; +const files = [{ to: 'private.key', generator: generatePrivateKey }]; function prepare() { fs.mkdirSync(SELF_HOST_CONFIG_DIR, { recursive: true }); - for (const { to, generator } of configFiles) { + for (const { to, generator } of files) { const targetFilePath = path.join(SELF_HOST_CONFIG_DIR, to); if (!fs.existsSync(targetFilePath)) { console.log(`creating config file [${targetFilePath}].`); diff --git a/packages/backend/server/src/__tests__/app/doc.e2e.ts b/packages/backend/server/src/__tests__/app/doc.e2e.ts deleted file mode 100644 index 0b0d89cc9a..0000000000 --- a/packages/backend/server/src/__tests__/app/doc.e2e.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { INestApplication } from '@nestjs/common'; -import type { TestFn } from 'ava'; -import ava from 'ava'; -import request from 'supertest'; - -import { buildAppModule } from '../../app.module'; -import { createTestingApp } from '../utils'; - -const test = ava as TestFn<{ - app: INestApplication; -}>; - -test.before('start app', async t => { - // @ts-expect-error override - AFFiNE.flavor = { - type: 'doc', - doc: true, - } as typeof AFFiNE.flavor; - const app = await createTestingApp({ - imports: [buildAppModule()], - }); - - t.context.app = app; -}); - -test.after.always(async t => { - await t.context.app.close(); -}); - -test('should init app', async t => { - const res = await request(t.context.app.getHttpServer()) - .get('/info') - .expect(200); - - t.is(res.body.flavor, 'doc'); -}); diff --git a/packages/backend/server/src/__tests__/app/graphql.e2e.ts b/packages/backend/server/src/__tests__/app/graphql.e2e.ts deleted file mode 100644 index e6916940aa..0000000000 --- a/packages/backend/server/src/__tests__/app/graphql.e2e.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { Args, Mutation, Resolver } from '@nestjs/graphql'; -import type { TestFn } from 'ava'; -import ava from 'ava'; -import GraphQLUpload, { - type FileUpload, -} from 'graphql-upload/GraphQLUpload.mjs'; -import request from 'supertest'; - -import { buildAppModule } from '../../app.module'; -import { Public } from '../../core/auth'; -import { createTestingApp, TestingApp } from '../utils'; - -const gql = '/graphql'; - -const test = ava as TestFn<{ - app: TestingApp; -}>; - -@Resolver(() => String) -class TestResolver { - @Public() - @Mutation(() => Number) - async upload( - @Args({ name: 'body', type: () => GraphQLUpload }) - body: FileUpload - ): Promise { - const size = await new Promise((resolve, reject) => { - const stream = body.createReadStream(); - let size = 0; - stream.on('data', chunk => (size += chunk.length)); - stream.on('error', reject); - stream.on('end', () => resolve(size)); - }); - - return size; - } -} - -test.before('start app', async t => { - // @ts-expect-error override - AFFiNE.flavor = { - type: 'graphql', - graphql: true, - } as typeof AFFiNE.flavor; - const app = await createTestingApp({ - imports: [buildAppModule()], - providers: [TestResolver], - }); - - t.context.app = app; -}); - -test.after.always(async t => { - await t.context.app.close(); -}); - -test('should init app', async t => { - await request(t.context.app.getHttpServer()) - .post(gql) - .send({ - query: ` - query { - error - } - `, - }) - .expect(400); - - const response = await request(t.context.app.getHttpServer()) - .post(gql) - .send({ - query: `query { - serverConfig { - name - version - type - features - } - }`, - }) - .expect(200); - - const config = response.body.data.serverConfig; - - t.is(config.type, 'Affine'); - t.true(Array.isArray(config.features)); - // make sure the request id is set - t.truthy(response.headers['x-request-id']); -}); - -test('should return 404 for unknown path', async t => { - await request(t.context.app.getHttpServer()).get('/unknown').expect(404); - - t.pass(); -}); - -test('should be able to call apis', async t => { - const res = await request(t.context.app.getHttpServer()) - .get('/info') - .expect(200); - - t.is(res.body.flavor, 'graphql'); - // make sure the request id is set - t.truthy(res.headers['x-request-id']); -}); - -test('should not throw internal error when graphql call with invalid params', async t => { - await t.throwsAsync(t.context.app.gql(`query { workspace("1") }`), { - message: /Failed to execute gql: query { workspace\("1"\) \}, status: 400/, - }); -}); - -test('should can send maximum size of body', async t => { - const { app } = t.context; - - const body = Buffer.from('a'.repeat(10 * 1024 * 1024 - 1)); - const res = await app - .POST('/graphql') - .set({ 'x-request-id': 'test', 'x-operation-name': 'test' }) - .field( - 'operations', - JSON.stringify({ - name: 'upload', - query: `mutation upload($body: Upload!) { upload(body: $body) }`, - variables: { body: null }, - }) - ) - .field('map', JSON.stringify({ '0': ['variables.body'] })) - .attach( - '0', - body, - `body-${Math.random().toString(16).substring(2, 10)}.data` - ) - .expect(200); - - t.is(Number(res.body.data.upload), body.length); -}); diff --git a/packages/backend/server/src/__tests__/app/renderer.e2e.ts b/packages/backend/server/src/__tests__/app/renderer.e2e.ts deleted file mode 100644 index 74057166e1..0000000000 --- a/packages/backend/server/src/__tests__/app/renderer.e2e.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { INestApplication } from '@nestjs/common'; -import type { TestFn } from 'ava'; -import ava from 'ava'; -import request from 'supertest'; - -import { buildAppModule } from '../../app.module'; -import { createTestingApp } from '../utils'; - -const test = ava as TestFn<{ - app: INestApplication; -}>; - -test.before('start app', async t => { - // @ts-expect-error override - AFFiNE.flavor = { - type: 'renderer', - renderer: true, - } as typeof AFFiNE.flavor; - const app = await createTestingApp({ - imports: [buildAppModule()], - }); - - t.context.app = app; -}); - -test.after.always(async t => { - await t.context.app.close(); -}); - -test('should init app', async t => { - const res = await request(t.context.app.getHttpServer()) - .get('/info') - .expect(200); - - t.is(res.body.flavor, 'renderer'); -}); diff --git a/packages/backend/server/src/__tests__/app/selfhost.e2e.ts b/packages/backend/server/src/__tests__/app/selfhost.e2e.ts index f894b7e60f..088dbdaaeb 100644 --- a/packages/backend/server/src/__tests__/app/selfhost.e2e.ts +++ b/packages/backend/server/src/__tests__/app/selfhost.e2e.ts @@ -8,7 +8,6 @@ import ava from 'ava'; import request from 'supertest'; import { buildAppModule } from '../../app.module'; -import { Config } from '../../base'; import { Public } from '../../core/auth'; import { ServerService } from '../../core/config'; import { createTestingApp, type TestingApp } from '../utils'; @@ -49,18 +48,16 @@ export class TestResolver { test.before('init selfhost server', async t => { // @ts-expect-error override - AFFiNE.isSelfhosted = true; - AFFiNE.flavor.renderer = true; + globalThis.env.DEPLOYMENT_TYPE = 'selfhosted'; const app = await createTestingApp({ - imports: [buildAppModule()], + imports: [buildAppModule(globalThis.env)], controllers: [TestResolver], }); t.context.app = app; t.context.db = t.context.app.get(PrismaClient); - const config = app.get(Config); - const staticPath = path.join(config.projectRoot, 'static'); + const staticPath = path.join(env.projectRoot, 'static'); initTestStaticFiles(staticPath); }); diff --git a/packages/backend/server/src/__tests__/app/sync.e2e.ts b/packages/backend/server/src/__tests__/app/sync.e2e.ts deleted file mode 100644 index f55c34d369..0000000000 --- a/packages/backend/server/src/__tests__/app/sync.e2e.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { INestApplication } from '@nestjs/common'; -import type { TestFn } from 'ava'; -import ava from 'ava'; -import request from 'supertest'; - -import { buildAppModule } from '../../app.module'; -import { createTestingApp } from '../utils'; - -const test = ava as TestFn<{ - app: INestApplication; -}>; - -test.before('start app', async t => { - // @ts-expect-error override - AFFiNE.flavor = { - type: 'sync', - sync: true, - } as typeof AFFiNE.flavor; - const app = await createTestingApp({ - imports: [buildAppModule()], - }); - - t.context.app = app; -}); - -test.after.always(async t => { - await t.context.app.close(); -}); - -test('should init app', async t => { - const res = await request(t.context.app.getHttpServer()) - .get('/info') - .expect(200); - - t.is(res.body.flavor, 'sync'); -}); diff --git a/packages/backend/server/src/__tests__/auth/controller.spec.ts b/packages/backend/server/src/__tests__/auth/controller.spec.ts index 82aba5a860..23f9b7e013 100644 --- a/packages/backend/server/src/__tests__/auth/controller.spec.ts +++ b/packages/backend/server/src/__tests__/auth/controller.spec.ts @@ -5,10 +5,7 @@ import { PrismaClient } from '@prisma/client'; import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; -import { AuthModule } from '../../core/auth'; import { AuthService } from '../../core/auth/service'; -import { FeatureModule } from '../../core/features'; -import { UserModule } from '../../core/user'; import { createTestingApp, currentUser, @@ -23,9 +20,7 @@ const test = ava as TestFn<{ }>; test.before(async t => { - const app = await createTestingApp({ - imports: [FeatureModule, UserModule, AuthModule], - }); + const app = await createTestingApp(); t.context.auth = app.get(AuthService); t.context.db = app.get(PrismaClient); diff --git a/packages/backend/server/src/__tests__/config.spec.ts b/packages/backend/server/src/__tests__/config.spec.ts deleted file mode 100644 index 3b3e71283d..0000000000 --- a/packages/backend/server/src/__tests__/config.spec.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { TestingModule } from '@nestjs/testing'; -import test from 'ava'; - -import { Config, ConfigModule } from '../base/config'; -import { createTestingModule } from './utils'; - -let config: Config; -let module: TestingModule; -test.beforeEach(async () => { - module = await createTestingModule({}, false); - config = module.get(Config); -}); - -test.afterEach.always(async () => { - await module.close(); -}); - -test('should be able to get config', t => { - t.true(typeof config.server.host === 'string'); - t.is(config.projectRoot, process.cwd()); - t.is(config.NODE_ENV, 'test'); -}); - -test('should be able to override config', async t => { - const module = await createTestingModule({ - imports: [ - ConfigModule.forRoot({ - server: { - host: 'testing', - }, - }), - ], - }); - const config = module.get(Config); - - t.is(config.server.host, 'testing'); - - await module.close(); -}); diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index 9c74aef19c..f8e18a297d 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -7,14 +7,7 @@ import { AuthService } from '../core/auth'; import { QuotaModule } from '../core/quota'; import { CopilotModule } from '../plugins/copilot'; import { prompts, PromptService } from '../plugins/copilot/prompt'; -import { - CopilotProviderService, - FalProvider, - OpenAIProvider, - PerplexityProvider, - registerCopilotProvider, - unregisterCopilotProvider, -} from '../plugins/copilot/providers'; +import { CopilotProviderFactory } from '../plugins/copilot/providers'; import { CopilotChatTextExecutor, CopilotWorkflowService, @@ -32,7 +25,7 @@ type Tester = { auth: AuthService; module: TestingModule; prompt: PromptService; - provider: CopilotProviderService; + factory: CopilotProviderFactory; workflow: CopilotWorkflowService; executors: { image: CopilotChatImageExecutor; @@ -67,9 +60,9 @@ const runIfCopilotConfigured = test.macro( test.serial.before(async t => { const module = await createTestingModule({ imports: [ - ConfigModule.forRoot({ - plugins: { - copilot: { + ConfigModule.override({ + copilot: { + providers: { openai: { apiKey: process.env.COPILOT_OPENAI_API_KEY, }, @@ -79,6 +72,9 @@ test.serial.before(async t => { perplexity: { apiKey: process.env.COPILOT_PERPLEXITY_API_KEY, }, + gemini: { + apiKey: process.env.COPILOT_GOOGLE_API_KEY, + }, }, }, }), @@ -89,13 +85,13 @@ test.serial.before(async t => { const auth = module.get(AuthService); const prompt = module.get(PromptService); - const provider = module.get(CopilotProviderService); + const factory = module.get(CopilotProviderFactory); const workflow = module.get(CopilotWorkflowService); t.context.module = module; t.context.auth = auth; t.context.prompt = prompt; - t.context.provider = provider; + t.context.factory = factory; t.context.workflow = workflow; t.context.executors = { image: module.get(CopilotChatImageExecutor), @@ -113,10 +109,6 @@ test.serial.before(async t => { executors.html.register(); executors.json.register(); - registerCopilotProvider(OpenAIProvider); - registerCopilotProvider(FalProvider); - registerCopilotProvider(PerplexityProvider); - for (const name of await prompt.listNames()) { await prompt.delete(name); } @@ -126,12 +118,6 @@ test.serial.before(async t => { } }); -test.after(async _ => { - unregisterCopilotProvider(OpenAIProvider.type); - unregisterCopilotProvider(FalProvider.type); - unregisterCopilotProvider(PerplexityProvider.type); -}); - test.after(async t => { await t.context.module.close(); }); @@ -523,12 +509,10 @@ for (const { name, promptName, messages, verifier, type } of actions) { `should be able to run action: ${promptName}${name ? ` - ${name}` : ''}`, runIfCopilotConfigured, async t => { - const { provider: providerService, prompt: promptService } = t.context; + const { factory, prompt: promptService } = t.context; const prompt = (await promptService.get(promptName))!; t.truthy(prompt, 'should have prompt'); - const provider = (await providerService.getProviderByModel( - prompt.model - ))!; + const provider = (await factory.getProviderByModel(prompt.model))!; t.truthy(provider, 'should have provider'); await retry(`action: ${promptName}`, t, async t => { if (type === 'text' && 'generateText' in provider) { diff --git a/packages/backend/server/src/__tests__/copilot.e2e.ts b/packages/backend/server/src/__tests__/copilot.e2e.ts index 1dbcf6f1e0..885c02173a 100644 --- a/packages/backend/server/src/__tests__/copilot.e2e.ts +++ b/packages/backend/server/src/__tests__/copilot.e2e.ts @@ -6,12 +6,11 @@ import type { TestFn } from 'ava'; import ava from 'ava'; import Sinon from 'sinon'; +import { AppModule } from '../app.module'; import { JobQueue } from '../base'; import { ConfigModule } from '../base/config'; import { AuthService } from '../core/auth'; import { DocReader } from '../core/doc'; -import { WorkspaceModule } from '../core/workspaces'; -import { CopilotModule } from '../plugins/copilot'; import { CopilotContextDocJob, CopilotContextService, @@ -19,14 +18,11 @@ import { import { MockEmbeddingClient } from '../plugins/copilot/context/embedding'; import { prompts, PromptService } from '../plugins/copilot/prompt'; import { - CopilotProviderService, - FalProvider, + CopilotProviderFactory, OpenAIProvider, - PerplexityProvider, - registerCopilotProvider, - unregisterCopilotProvider, } from '../plugins/copilot/providers'; import { CopilotStorage } from '../plugins/copilot/storage'; +import { MockCopilotProvider } from './mocks'; import { acceptInviteById, createTestingApp, @@ -53,7 +49,6 @@ import { listContextDocAndFiles, matchFiles, matchWorkspaceDocs, - MockCopilotTestProvider, sse2array, textToEventStream, unsplashSearch, @@ -67,7 +62,7 @@ const test = ava as TestFn<{ context: CopilotContextService; jobs: CopilotContextDocJob; prompt: PromptService; - provider: CopilotProviderService; + factory: CopilotProviderFactory; storage: CopilotStorage; u1: TestUser; }>; @@ -75,24 +70,19 @@ const test = ava as TestFn<{ test.before(async t => { const app = await createTestingApp({ imports: [ - ConfigModule.forRoot({ - plugins: { - copilot: { - openai: { - apiKey: '1', - }, - fal: { - apiKey: '1', - }, - perplexity: { - apiKey: '1', - }, - unsplashKey: process.env.UNSPLASH_ACCESS_KEY || '1', + ConfigModule.override({ + copilot: { + providers: { + openai: { apiKey: '1' }, + fal: {}, + perplexity: {}, + }, + unsplash: { + key: process.env.UNSPLASH_ACCESS_KEY || '1', }, }, }), - WorkspaceModule, - CopilotModule, + AppModule, ], tapModule: m => { // use real JobQueue for testing @@ -105,6 +95,7 @@ test.before(async t => { }; }, }); + m.overrideProvider(OpenAIProvider).useClass(MockCopilotProvider); }, }); @@ -129,14 +120,9 @@ test.beforeEach(async t => { Sinon.restore(); const { app, prompt } = t.context; await app.initTestingDB(); - await prompt.onModuleInit(); + await prompt.onApplicationBootstrap(); t.context.u1 = await app.signupV1('u1@affine.pro'); - unregisterCopilotProvider(OpenAIProvider.type); - unregisterCopilotProvider(FalProvider.type); - unregisterCopilotProvider(PerplexityProvider.type); - registerCopilotProvider(MockCopilotTestProvider); - await prompt.set(promptName, 'test', [ { role: 'system', content: 'hello {{word}}' }, ]); @@ -761,13 +747,12 @@ test('should be able to manage context', async t => { 'should throw error if create context with invalid session id' ); - const context = createCopilotContext(app, workspaceId, sessionId); - await t.notThrowsAsync(context, 'should create context with chat session'); + const context = await createCopilotContext(app, workspaceId, sessionId); const list = await listContext(app, workspaceId, sessionId); t.deepEqual( list.map(f => ({ id: f.id })), - [{ id: await context }], + [{ id: context }], 'should list context' ); } diff --git a/packages/backend/server/src/__tests__/copilot.spec.ts b/packages/backend/server/src/__tests__/copilot.spec.ts index f559ca8239..30ac64ede3 100644 --- a/packages/backend/server/src/__tests__/copilot.spec.ts +++ b/packages/backend/server/src/__tests__/copilot.spec.ts @@ -19,18 +19,14 @@ import { import { MockEmbeddingClient } from '../plugins/copilot/context/embedding'; import { prompts, PromptService } from '../plugins/copilot/prompt'; import { - CopilotProviderService, + CopilotCapability, + CopilotProviderFactory, + CopilotProviderType, OpenAIProvider, - registerCopilotProvider, - unregisterCopilotProvider, } from '../plugins/copilot/providers'; import { CitationParser } from '../plugins/copilot/providers/perplexity'; import { ChatSessionService } from '../plugins/copilot/session'; import { CopilotStorage } from '../plugins/copilot/storage'; -import { - CopilotCapability, - CopilotProviderType, -} from '../plugins/copilot/types'; import { CopilotChatTextExecutor, CopilotWorkflowService, @@ -50,8 +46,9 @@ import { } from '../plugins/copilot/workflow/executor'; import { AutoRegisteredWorkflowExecutor } from '../plugins/copilot/workflow/executor/utils'; import { WorkflowGraphList } from '../plugins/copilot/workflow/graph'; +import { MockCopilotProvider } from './mocks'; import { createTestingModule, TestingModule } from './utils'; -import { MockCopilotTestProvider, WorkflowTestCases } from './utils/copilot'; +import { WorkflowTestCases } from './utils/copilot'; const test = ava as TestFn<{ auth: AuthService; @@ -60,7 +57,7 @@ const test = ava as TestFn<{ event: EventBus; context: CopilotContextService; prompt: PromptService; - provider: CopilotProviderService; + factory: CopilotProviderFactory; session: ChatSessionService; jobs: CopilotContextDocJob; storage: CopilotStorage; @@ -77,9 +74,9 @@ let userId: string; test.before(async t => { const module = await createTestingModule({ imports: [ - ConfigModule.forRoot({ - plugins: { - copilot: { + ConfigModule.override({ + copilot: { + providers: { openai: { apiKey: process.env.COPILOT_OPENAI_API_KEY ?? '1', }, @@ -95,6 +92,9 @@ test.before(async t => { QuotaModule, CopilotModule, ], + tapModule: builder => { + builder.overrideProvider(OpenAIProvider).useClass(MockCopilotProvider); + }, }); const auth = module.get(AuthService); @@ -102,7 +102,7 @@ test.before(async t => { const event = module.get(EventBus); const context = module.get(CopilotContextService); const prompt = module.get(PromptService); - const provider = module.get(CopilotProviderService); + const factory = module.get(CopilotProviderFactory); const session = module.get(ChatSessionService); const workflow = module.get(CopilotWorkflowService); const jobs = module.get(CopilotContextDocJob); @@ -114,7 +114,7 @@ test.before(async t => { t.context.event = event; t.context.context = context; t.context.prompt = prompt; - t.context.provider = provider; + t.context.factory = factory; t.context.session = session; t.context.workflow = workflow; t.context.jobs = jobs; @@ -131,7 +131,7 @@ test.beforeEach(async t => { Sinon.restore(); const { module, auth, prompt } = t.context; await module.initTestingDB(); - await prompt.onModuleInit(); + await prompt.onApplicationBootstrap(); const user = await auth.signUp('test@affine.pro', '123456'); userId = user.id; }); @@ -730,10 +730,10 @@ test('should handle params correctly in chat session', async t => { // ==================== provider ==================== test('should be able to get provider', async t => { - const { provider } = t.context; + const { factory } = t.context; { - const p = await provider.getProviderByCapability( + const p = await factory.getProviderByCapability( CopilotCapability.TextToText ); t.is( @@ -744,108 +744,40 @@ test('should be able to get provider', async t => { } { - const p = await provider.getProviderByCapability( - CopilotCapability.TextToEmbedding + const p = await factory.getProviderByCapability( + CopilotCapability.ImageToImage, + { model: 'lora/image-to-image' } ); t.is( p?.type.toString(), - 'openai', + 'fal', 'should get provider support text-to-embedding' ); } { - const p = await provider.getProviderByCapability( - CopilotCapability.TextToImage - ); - t.is( - p?.type.toString(), - 'fal', - 'should get provider support text-to-image' - ); - } - - { - const p = await provider.getProviderByCapability( - CopilotCapability.ImageToImage - ); - t.is( - p?.type.toString(), - 'fal', - 'should get provider support image-to-image' - ); - } - - { - const p = await provider.getProviderByCapability( - CopilotCapability.ImageToText - ); - t.is( - p?.type.toString(), - 'fal', - 'should get provider support image-to-text' - ); - } - - // text-to-image use fal by default, but this case can use - // model dall-e-3 to select openai provider - { - const p = await provider.getProviderByCapability( - CopilotCapability.TextToImage, - 'dall-e-3' - ); - t.is( - p?.type.toString(), - 'openai', - 'should get provider support text-to-image and model' - ); - } - - // gpt4o is not defined now, but it already published by openai - // we should check from online api if it is available - { - const p = await provider.getProviderByCapability( + const p = await factory.getProviderByCapability( CopilotCapability.ImageToText, - 'gpt-4o-2024-08-06' + { prefer: CopilotProviderType.FAL } ); t.is( p?.type.toString(), - 'openai', - 'should get provider support text-to-image and model' + 'fal', + 'should get provider support text-to-embedding' ); } // if a model is not defined and not available in online api // it should return null { - const p = await provider.getProviderByCapability( + const p = await factory.getProviderByCapability( CopilotCapability.ImageToText, - 'gpt-4-not-exist' + { model: 'gpt-4-not-exist' } ); t.falsy(p, 'should not get provider'); } }); -test('should be able to register test provider', async t => { - const { provider } = t.context; - registerCopilotProvider(MockCopilotTestProvider); - - const assertProvider = async (cap: CopilotCapability) => { - const p = await provider.getProviderByCapability(cap, 'test'); - t.is( - p?.type, - CopilotProviderType.Test, - `should get test provider with ${cap}` - ); - }; - - await assertProvider(CopilotCapability.TextToText); - await assertProvider(CopilotCapability.TextToEmbedding); - await assertProvider(CopilotCapability.TextToImage); - await assertProvider(CopilotCapability.ImageToImage); - await assertProvider(CopilotCapability.ImageToText); -}); - // ==================== workflow ==================== // this test used to preview the final result of the workflow @@ -854,7 +786,6 @@ test.skip('should be able to preview workflow', async t => { const { prompt, workflow, executors } = t.context; executors.text.register(); - registerCopilotProvider(OpenAIProvider); for (const p of prompts) { await prompt.set(p.name, p.model, p.messages, p.config); @@ -878,8 +809,6 @@ test.skip('should be able to preview workflow', async t => { } console.log('final stream result:', result); t.truthy(result, 'should return result'); - - unregisterCopilotProvider(OpenAIProvider.type); }); const runWorkflow = async function* runWorkflow( @@ -900,8 +829,6 @@ test('should be able to run pre defined workflow', async t => { executors.text.register(); executors.html.register(); executors.json.register(); - unregisterCopilotProvider(OpenAIProvider.type); - registerCopilotProvider(MockCopilotTestProvider); const executor = Sinon.spy(executors.text, 'next'); @@ -941,17 +868,12 @@ test('should be able to run pre defined workflow', async t => { } } } - - unregisterCopilotProvider(MockCopilotTestProvider.type); - registerCopilotProvider(OpenAIProvider); }); test('should be able to run workflow', async t => { const { workflow, executors } = t.context; executors.text.register(); - unregisterCopilotProvider(OpenAIProvider.type); - registerCopilotProvider(MockCopilotTestProvider); const executor = Sinon.spy(executors.text, 'next'); @@ -998,9 +920,6 @@ test('should be able to run workflow', async t => { 'graph params should correct' ); } - - unregisterCopilotProvider(MockCopilotTestProvider.type); - registerCopilotProvider(OpenAIProvider); }); // ==================== workflow executor ==================== @@ -1037,18 +956,16 @@ test('should be able to run executor', async t => { }); test('should be able to run text executor', async t => { - const { executors, provider, prompt } = t.context; + const { executors, factory, prompt } = t.context; executors.text.register(); const executor = getWorkflowExecutor(executors.text.type); - unregisterCopilotProvider(OpenAIProvider.type); - registerCopilotProvider(MockCopilotTestProvider); await prompt.set('test', 'test', [ { role: 'system', content: 'hello {{word}}' }, ]); // mock provider const testProvider = - (await provider.getProviderByModel('test'))!; + (await factory.getProviderByModel('test'))!; const text = Sinon.spy(testProvider, 'generateText'); const textStream = Sinon.spy(testProvider, 'generateTextStream'); @@ -1103,23 +1020,19 @@ test('should be able to run text executor', async t => { } Sinon.restore(); - unregisterCopilotProvider(MockCopilotTestProvider.type); - registerCopilotProvider(OpenAIProvider); }); test('should be able to run image executor', async t => { - const { executors, provider, prompt } = t.context; + const { executors, factory, prompt } = t.context; executors.image.register(); const executor = getWorkflowExecutor(executors.image.type); - unregisterCopilotProvider(OpenAIProvider.type); - registerCopilotProvider(MockCopilotTestProvider); await prompt.set('test', 'test', [ { role: 'user', content: 'tag1, tag2, tag3, {{#tags}}{{.}}, {{/tags}}' }, ]); // mock provider const testProvider = - (await provider.getProviderByModel('test'))!; + (await factory.getProviderByModel('test'))!; const image = Sinon.spy(testProvider, 'generateImages'); const imageStream = Sinon.spy(testProvider, 'generateImagesStream'); @@ -1184,8 +1097,6 @@ test('should be able to run image executor', async t => { } Sinon.restore(); - unregisterCopilotProvider(MockCopilotTestProvider.type); - registerCopilotProvider(OpenAIProvider); }); test('CitationParser should replace citation placeholders with URLs', t => { diff --git a/packages/backend/server/src/__tests__/create-module.ts b/packages/backend/server/src/__tests__/create-module.ts index 3d7e40d283..2b90525c52 100644 --- a/packages/backend/server/src/__tests__/create-module.ts +++ b/packages/backend/server/src/__tests__/create-module.ts @@ -4,9 +4,11 @@ import { TestingModule as NestjsTestingModule, TestingModuleBuilder, } from '@nestjs/testing'; +import { PrismaClient } from '@prisma/client'; import { FunctionalityModules } from '../app.module'; -import { AFFiNELogger } from '../base'; +import { AFFiNELogger, EventBus, JobQueue } from '../base'; +import { createFactory, MockEventBus, MockJobQueue } from './mocks'; import { TEST_LOG_LEVEL } from './utils'; interface TestingModuleMetadata extends ModuleMetadata { @@ -15,10 +17,13 @@ interface TestingModuleMetadata extends ModuleMetadata { export interface TestingModule extends NestjsTestingModule { [Symbol.asyncDispose](): Promise; + create: ReturnType; + queue: MockJobQueue; + event: MockEventBus; } export async function createModule( - metadata: TestingModuleMetadata + metadata: TestingModuleMetadata = {} ): Promise { const { tapModule, ...meta } = metadata; @@ -27,6 +32,12 @@ export async function createModule( imports: [...FunctionalityModules, ...(meta.imports ?? [])], }); + builder + .overrideProvider(JobQueue) + .useValue(new MockJobQueue()) + .overrideProvider(EventBus) + .useValue(new MockEventBus()); + // when custom override happens if (tapModule) { tapModule(builder); @@ -44,6 +55,9 @@ export async function createModule( module[Symbol.asyncDispose] = async () => { await module.close(); }; + module.create = createFactory(module.get(PrismaClient)); + module.queue = module.get(JobQueue); + module.event = module.get(EventBus); return module; } diff --git a/packages/backend/server/src/__tests__/doc/renderer.spec.ts b/packages/backend/server/src/__tests__/doc/renderer.spec.ts index 41710d6ffc..8189cbcec1 100644 --- a/packages/backend/server/src/__tests__/doc/renderer.spec.ts +++ b/packages/backend/server/src/__tests__/doc/renderer.spec.ts @@ -7,7 +7,6 @@ import type { TestFn } from 'ava'; import ava from 'ava'; import request from 'supertest'; -import { DocRendererModule } from '../../core/doc-renderer'; import { createTestingApp } from '../utils'; const test = ava as TestFn<{ @@ -45,13 +44,11 @@ function initTestStaticFiles(staticPath: string) { } } -test.before('init selfhost server', async t => { +test.before(async t => { const staticPath = new Package('@affine/server').join('static').value; initTestStaticFiles(staticPath); - const app = await createTestingApp({ - imports: [DocRendererModule], - }); + const app = await createTestingApp(); t.context.app = app; }); diff --git a/packages/backend/server/src/__tests__/e2e/app.spec.ts b/packages/backend/server/src/__tests__/e2e/apps/app.spec.ts similarity index 77% rename from packages/backend/server/src/__tests__/e2e/app.spec.ts rename to packages/backend/server/src/__tests__/e2e/apps/app.spec.ts index b66696a97d..45f89275e3 100644 --- a/packages/backend/server/src/__tests__/e2e/app.spec.ts +++ b/packages/backend/server/src/__tests__/e2e/apps/app.spec.ts @@ -1,7 +1,7 @@ import { getCurrentUserQuery } from '@affine/graphql'; -import { Mockers } from '../mocks'; -import { app, e2e } from './test'; +import { Mockers } from '../../mocks'; +import { app, e2e } from '../test'; e2e('should create test app correctly', async t => { t.truthy(app); @@ -18,12 +18,7 @@ e2e('should mock queue work', async t => { e2e('should handle http request', async t => { const res = await app.GET('/info'); t.is(res.status, 200); - t.is(res.body.compatibility, AFFiNE.version); -}); - -e2e('should handle gql request', async t => { - const user = await app.gql({ query: getCurrentUserQuery }); - t.is(user.currentUser, null); + t.is(res.body.compatibility, env.version); }); e2e('should create workspace with owner', async t => { diff --git a/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts b/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts new file mode 100644 index 0000000000..f1ae94a3a8 --- /dev/null +++ b/packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts @@ -0,0 +1,46 @@ +import { getCurrentUserQuery } from '@affine/graphql'; + +import { createApp } from '../create-app'; +import { e2e } from '../test'; + +e2e('should init doc service', async t => { + // @ts-expect-error override + globalThis.env.FLAVOR = 'doc'; + await using app = await createApp(); + + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'doc'); + + await t.throwsAsync(app.gql({ query: getCurrentUserQuery })); +}); + +e2e('should init graphql service', async t => { + // @ts-expect-error override + globalThis.env.FLAVOR = 'graphql'; + await using app = await createApp(); + + const res = await app.GET('/info').expect(200); + + t.is(res.body.flavor, 'graphql'); + + const user = await app.gql({ query: getCurrentUserQuery }); + t.is(user.currentUser, null); +}); + +e2e('should init sync service', async t => { + // @ts-expect-error override + globalThis.env.FLAVOR = 'sync'; + await using app = await createApp(); + + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'sync'); +}); + +e2e('should init renderer service', async t => { + // @ts-expect-error override + globalThis.env.FLAVOR = 'renderer'; + await using app = await createApp(); + + const res = await app.GET('/info').expect(200); + t.is(res.body.flavor, 'renderer'); +}); diff --git a/packages/backend/server/src/__tests__/e2e/create-app.ts b/packages/backend/server/src/__tests__/e2e/create-app.ts index d8219b62b5..db3f47680e 100644 --- a/packages/backend/server/src/__tests__/e2e/create-app.ts +++ b/packages/backend/server/src/__tests__/e2e/create-app.ts @@ -13,6 +13,7 @@ import { AFFiNELogger, CacheInterceptor, CloudThrottlerGuard, + EventBus, GlobalExceptionFilter, JobQueue, OneMB, @@ -23,6 +24,7 @@ import { Mailer } from '../../core/mail'; import { createFactory, MockedUser, + MockEventBus, MockJobQueue, MockMailer, MockUser, @@ -181,23 +183,19 @@ export class TestingApp extends NestApplication { } } -let GLOBAL_APP_INSTANCE: TestingApp | null = null; export async function createApp( metadata: TestingAppMetadata = {} ): Promise { - if (GLOBAL_APP_INSTANCE) { - return GLOBAL_APP_INSTANCE; - } - const { buildAppModule } = await import('../../app.module'); const { tapModule, tapApp } = metadata; const builder = Test.createTestingModule({ - imports: [buildAppModule()], + imports: [buildAppModule(globalThis.env)], }); builder.overrideProvider(Mailer).useValue(new MockMailer()); builder.overrideProvider(JobQueue).useValue(new MockJobQueue()); + builder.overrideProvider(EventBus).useValue(new MockEventBus()); // when custom override happens if (tapModule) { @@ -240,6 +238,5 @@ export async function createApp( await app.init(); - GLOBAL_APP_INSTANCE = app; return app; } diff --git a/packages/backend/server/src/__tests__/env.spec.ts b/packages/backend/server/src/__tests__/env.spec.ts new file mode 100644 index 0000000000..cd7f317dcf --- /dev/null +++ b/packages/backend/server/src/__tests__/env.spec.ts @@ -0,0 +1,156 @@ +import test from 'ava'; + +import { Env } from '../env'; + +const envs = { ...process.env }; +test.beforeEach(() => { + process.env = { ...envs }; +}); + +test('should init env', t => { + t.true(globalThis.env.testing); +}); + +test('should read NODE_ENV', t => { + process.env.NODE_ENV = 'test'; + t.deepEqual( + ['test', 'development', 'production'].map(envVal => { + process.env.NODE_ENV = envVal; + const env = new Env(); + return env.NODE_ENV; + }), + ['test', 'development', 'production'] + ); + + t.throws( + () => { + process.env.NODE_ENV = 'unknown'; + new Env(); + }, + { + message: + 'Invalid value "unknown" for environment variable NODE_ENV, expected one of ["development","test","production"]', + } + ); +}); + +test('should read NAMESPACE', t => { + t.deepEqual( + ['dev', 'beta', 'production'].map(envVal => { + process.env.AFFINE_ENV = envVal; + const env = new Env(); + return env.NAMESPACE; + }), + ['dev', 'beta', 'production'] + ); + + t.throws(() => { + process.env.AFFINE_ENV = 'unknown'; + new Env(); + }); +}); + +test('should read DEPLOYMENT_TYPE', t => { + t.deepEqual( + ['affine', 'selfhosted'].map(envVal => { + process.env.DEPLOYMENT_TYPE = envVal; + const env = new Env(); + return env.DEPLOYMENT_TYPE; + }), + ['affine', 'selfhosted'] + ); + + t.throws(() => { + process.env.DEPLOYMENT_TYPE = 'unknown'; + new Env(); + }); +}); + +test('should read FLAVOR', t => { + t.deepEqual( + ['allinone', 'graphql', 'sync', 'renderer', 'doc', 'script'].map(envVal => { + process.env.SERVER_FLAVOR = envVal; + const env = new Env(); + return env.FLAVOR; + }), + ['allinone', 'graphql', 'sync', 'renderer', 'doc', 'script'] + ); + + t.throws( + () => { + process.env.SERVER_FLAVOR = 'unknown'; + new Env(); + }, + { + message: + 'Invalid value "unknown" for environment variable SERVER_FLAVOR, expected one of ["allinone","graphql","sync","renderer","doc","script"]', + } + ); +}); + +test('should read platform', t => { + t.deepEqual( + ['gcp', 'unknown'].map(envVal => { + process.env.DEPLOYMENT_PLATFORM = envVal; + const env = new Env(); + return env.platform; + }), + ['gcp', 'unknown'] + ); + + t.notThrows(() => { + process.env.PLATFORM = 'unknown'; + new Env(); + }); +}); + +test('should tell flavors correctly', t => { + process.env.SERVER_FLAVOR = 'allinone'; + t.deepEqual(new Env().flavors, { + graphql: true, + sync: true, + renderer: true, + doc: true, + script: true, + }); + + process.env.SERVER_FLAVOR = 'graphql'; + t.deepEqual(new Env().flavors, { + graphql: true, + sync: false, + renderer: false, + doc: false, + script: false, + }); +}); + +test('should tell selfhosted correctly', t => { + process.env.DEPLOYMENT_TYPE = 'selfhosted'; + t.true(new Env().selfhosted); + + process.env.DEPLOYMENT_TYPE = 'affine'; + t.false(new Env().selfhosted); +}); + +test('should tell namespaces correctly', t => { + process.env.AFFINE_ENV = 'dev'; + t.deepEqual(new Env().namespaces, { + canary: true, + beta: false, + production: false, + }); + + process.env.AFFINE_ENV = 'beta'; + t.deepEqual(new Env().namespaces, { + canary: false, + beta: true, + production: false, + }); + + process.env.AFFINE_ENV = 'production'; + t.deepEqual(new Env().namespaces, { + canary: false, + beta: false, + production: true, + }); +}); diff --git a/packages/backend/server/src/__tests__/mocks/copilot.mock.ts b/packages/backend/server/src/__tests__/mocks/copilot.mock.ts new file mode 100644 index 0000000000..751b03a97f --- /dev/null +++ b/packages/backend/server/src/__tests__/mocks/copilot.mock.ts @@ -0,0 +1,113 @@ +import { randomBytes } from 'node:crypto'; + +import { + CopilotCapability, + CopilotChatOptions, + CopilotEmbeddingOptions, + PromptMessage, +} from '../../plugins/copilot/providers'; +import { + DEFAULT_DIMENSIONS, + OpenAIProvider, +} from '../../plugins/copilot/providers/openai'; +import { sleep } from '../utils/utils'; + +export class MockCopilotProvider extends OpenAIProvider { + override readonly models = [ + 'test', + 'gpt-4o', + 'gpt-4o-2024-08-06', + 'fast-sdxl/image-to-image', + 'lcm-sd15-i2i', + 'clarity-upscaler', + 'imageutils/rembg', + ]; + + override readonly capabilities = [ + CopilotCapability.TextToText, + CopilotCapability.TextToEmbedding, + CopilotCapability.TextToImage, + CopilotCapability.ImageToImage, + CopilotCapability.ImageToText, + ]; + + // ====== text to text ====== + + override async generateText( + messages: PromptMessage[], + model: string = 'test', + options: CopilotChatOptions = {} + ): Promise { + this.checkParams({ messages, model, options }); + // make some time gap for history test case + await sleep(100); + return 'generate text to text'; + } + + override async *generateTextStream( + messages: PromptMessage[], + model: string = 'gpt-4o-mini', + options: CopilotChatOptions = {} + ): AsyncIterable { + this.checkParams({ messages, model, options }); + + // make some time gap for history test case + await sleep(100); + const result = 'generate text to text stream'; + for (const message of result) { + yield message; + if (options.signal?.aborted) { + break; + } + } + } + + // ====== text to embedding ====== + + override async generateEmbedding( + messages: string | string[], + model: string, + options: CopilotEmbeddingOptions = { dimensions: DEFAULT_DIMENSIONS } + ): Promise { + messages = Array.isArray(messages) ? messages : [messages]; + this.checkParams({ embeddings: messages, model, options }); + + // make some time gap for history test case + await sleep(100); + return [Array.from(randomBytes(options.dimensions)).map(v => v % 128)]; + } + + // ====== text to image ====== + override async generateImages( + messages: PromptMessage[], + model: string = 'test', + _options: { + signal?: AbortSignal; + user?: string; + } = {} + ): Promise> { + const { content: prompt } = messages[0] || {}; + if (!prompt) { + throw new Error('Prompt is required'); + } + + // make some time gap for history test case + await sleep(100); + // just let test case can easily verify the final prompt + return [`https://example.com/${model}.jpg`, prompt]; + } + + override async *generateImagesStream( + messages: PromptMessage[], + model: string = 'dall-e-3', + options: { + signal?: AbortSignal; + user?: string; + } = {} + ): AsyncIterable { + const ret = await this.generateImages(messages, model, options); + for (const url of ret) { + yield url; + } + } +} diff --git a/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts b/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts new file mode 100644 index 0000000000..c9f887b22f --- /dev/null +++ b/packages/backend/server/src/__tests__/mocks/eventbus.mock.ts @@ -0,0 +1,35 @@ +import Sinon from 'sinon'; + +import { EventBus } from '../../base'; +import { EventName } from '../../base/event/def'; + +export class MockEventBus { + private readonly stub = Sinon.createStubInstance(EventBus); + + emit = this.stub.emitAsync; + emitAsync = this.stub.emitAsync; + broadcast = this.stub.broadcast; + + last( + name: Event + ): { name: Event; payload: Events[Event] } { + const call = this.emitAsync + .getCalls() + .reverse() + .find(call => call.args[0] === name); + if (!call) { + throw new Error(`Event ${name} never called`); + } + + // @ts-expect-error allow + return { + name, + payload: call.args[1], + }; + } + + count(name: EventName) { + return this.emitAsync.getCalls().filter(call => call.args[0] === name) + .length; + } +} diff --git a/packages/backend/server/src/__tests__/mocks/index.ts b/packages/backend/server/src/__tests__/mocks/index.ts index 785103b862..50e82b334a 100644 --- a/packages/backend/server/src/__tests__/mocks/index.ts +++ b/packages/backend/server/src/__tests__/mocks/index.ts @@ -4,7 +4,9 @@ export * from './user.mock'; export * from './workspace.mock'; export * from './workspace-user.mock'; +import { MockCopilotProvider } from './copilot.mock'; import { MockDocMeta } from './doc-meta.mock'; +import { MockEventBus } from './eventbus.mock'; import { MockMailer } from './mailer.mock'; import { MockJobQueue } from './queue.mock'; import { MockTeamWorkspace } from './team-workspace.mock'; @@ -22,4 +24,4 @@ export const Mockers = { DocMeta: MockDocMeta, }; -export { MockJobQueue, MockMailer }; +export { MockCopilotProvider, MockEventBus, MockJobQueue, MockMailer }; diff --git a/packages/backend/server/src/__tests__/models/feature-user.spec.ts b/packages/backend/server/src/__tests__/models/feature-user.spec.ts index e9a788807d..890f0e199f 100644 --- a/packages/backend/server/src/__tests__/models/feature-user.spec.ts +++ b/packages/backend/server/src/__tests__/models/feature-user.spec.ts @@ -1,7 +1,6 @@ import { User } from '@prisma/client'; import ava, { TestFn } from 'ava'; -import { ConfigModule } from '../../base/config'; import { FeatureType, Models, UserFeatureModel, UserModel } from '../../models'; import { createTestingModule, TestingModule } from '../utils'; @@ -126,13 +125,9 @@ test('should not switch user quota if the new quota is the same as the current o }); test('should use pro plan as free for selfhost instance', async t => { - await using module = await createTestingModule({ - imports: [ - ConfigModule.forRoot({ - isSelfhosted: true, - }), - ], - }); + // @ts-expect-error + env.DEPLOYMENT_TYPE = 'selfhosted'; + await using module = await createTestingModule(); const models = module.get(Models); const u1 = await models.user.create({ diff --git a/packages/backend/server/src/__tests__/nestjs/throttler.spec.ts b/packages/backend/server/src/__tests__/nestjs/throttler.spec.ts index 4b01b07647..e535ac2747 100644 --- a/packages/backend/server/src/__tests__/nestjs/throttler.spec.ts +++ b/packages/backend/server/src/__tests__/nestjs/throttler.spec.ts @@ -1,5 +1,3 @@ -import '../../plugins/config'; - import { Controller, Get, HttpStatus, UseGuards } from '@nestjs/common'; import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; @@ -89,11 +87,13 @@ class NonThrottledController { test.before(async t => { const app = await createTestingApp({ imports: [ - ConfigModule.forRoot({ - throttler: { - default: { - ttl: 60, - limit: 120, + ConfigModule.override({ + throttle: { + throttlers: { + default: { + ttl: 60, + limit: 120, + }, }, }, }), diff --git a/packages/backend/server/src/__tests__/oauth/controller.spec.ts b/packages/backend/server/src/__tests__/oauth/controller.spec.ts index 4205b8a67d..28f86bafdd 100644 --- a/packages/backend/server/src/__tests__/oauth/controller.spec.ts +++ b/packages/backend/server/src/__tests__/oauth/controller.spec.ts @@ -1,5 +1,3 @@ -import '../../plugins/config'; - import { randomUUID } from 'node:crypto'; import { HttpStatus } from '@nestjs/common'; @@ -30,14 +28,12 @@ const test = ava as TestFn<{ test.before(async t => { const app = await createTestingApp({ imports: [ - ConfigModule.forRoot({ - plugins: { - oauth: { - providers: { - google: { - clientId: 'google-client-id', - clientSecret: 'google-client-secret', - }, + ConfigModule.override({ + oauth: { + providers: { + google: { + clientId: 'google-client-id', + clientSecret: 'google-client-secret', }, }, }, diff --git a/packages/backend/server/src/__tests__/payment/service.spec.ts b/packages/backend/server/src/__tests__/payment/service.spec.ts index 673a001355..a12b176f94 100644 --- a/packages/backend/server/src/__tests__/payment/service.spec.ts +++ b/packages/backend/server/src/__tests__/payment/service.spec.ts @@ -6,12 +6,13 @@ import Sinon from 'sinon'; import Stripe from 'stripe'; import { AppModule } from '../../app.module'; -import { EventBus, Runtime } from '../../base'; -import { ConfigModule } from '../../base/config'; +import { EventBus } from '../../base'; +import { ConfigFactory, ConfigModule } from '../../base/config'; import { CurrentUser } from '../../core/auth'; import { AuthService } from '../../core/auth/service'; import { EarlyAccessType, FeatureService } from '../../core/features'; import { SubscriptionService } from '../../plugins/payment/service'; +import { StripeFactory } from '../../plugins/payment/stripe'; import { CouponType, encodeLookupKey, @@ -159,7 +160,6 @@ const test = ava as TestFn<{ service: SubscriptionService; event: Sinon.SinonStubbedInstance; feature: Sinon.SinonStubbedInstance; - runtime: Sinon.SinonStubbedInstance; stripe: { customers: Sinon.SinonStubbedInstance; prices: Sinon.SinonStubbedInstance; @@ -184,16 +184,12 @@ function getLastCheckoutPrice(checkoutStub: Sinon.SinonStub) { test.before(async t => { const app = await createTestingApp({ imports: [ - ConfigModule.forRoot({ - plugins: { - payment: { - stripe: { - keys: { - APIKey: '1', - webhookKey: '1', - }, - }, - }, + ConfigModule.override({ + payment: { + enabled: true, + showLifetimePrice: true, + apiKey: '1', + webhookKey: '1', }, }), AppModule, @@ -203,18 +199,19 @@ test.before(async t => { Sinon.createStubInstance(FeatureService) ); m.overrideProvider(EventBus).useValue(Sinon.createStubInstance(EventBus)); - m.overrideProvider(Runtime).useValue(Sinon.createStubInstance(Runtime)); }, }); t.context.event = app.get(EventBus); t.context.service = app.get(SubscriptionService); t.context.feature = app.get(FeatureService); - t.context.runtime = app.get(Runtime); t.context.db = app.get(PrismaClient); t.context.app = app; - const stripe = app.get(Stripe); + const stripeFactory = app.get(StripeFactory); + await stripeFactory.onConfigInit(); + + const stripe = stripeFactory.stripe; const stripeStubs = { customers: Sinon.stub(stripe.customers), prices: Sinon.stub(stripe.prices), @@ -234,6 +231,12 @@ test.beforeEach(async t => { await t.context.app.initTestingDB(); t.context.u1 = await app.get(AuthService).signUp('u1@affine.pro', '1'); + app.get(ConfigFactory).override({ + payment: { + showLifetimePrice: true, + }, + }); + await db.workspace.create({ data: { id: 'ws_1', @@ -249,11 +252,6 @@ test.beforeEach(async t => { Sinon.reset(); - // default stubs - t.context.runtime.fetch - .withArgs('plugins.payment/showLifetimePrice') - .resolves(true); - // @ts-expect-error stub stripe.prices.list.callsFake((params: Stripe.PriceListParams) => { if (params.lookup_keys) { @@ -294,8 +292,13 @@ test('should list normal prices for authenticated user', async t => { }); test('should not show lifetime price if not enabled', async t => { - const { service, runtime } = t.context; - runtime.fetch.withArgs('plugins.payment/showLifetimePrice').resolves(false); + const { service, app } = t.context; + + app.get(ConfigFactory).override({ + payment: { + showLifetimePrice: false, + }, + }); const prices = await service.listPrices(t.context.u1); @@ -539,8 +542,11 @@ test('should get correct pro plan price for checking out', async t => { // any user, lifetime recurring { feature.isEarlyAccessUser.resolves(false); - const runtime = app.get(Runtime); - await runtime.set('plugins.payment/showLifetimePrice', true); + app.get(ConfigFactory).override({ + payment: { + showLifetimePrice: true, + }, + }); await service.checkout( { @@ -1181,8 +1187,12 @@ const onetimeYearlyInvoice: Stripe.Invoice = { }; test('should not be able to checkout for lifetime recurring if not enabled', async t => { - const { service, u1, runtime } = t.context; - runtime.fetch.withArgs('plugins.payment/showLifetimePrice').resolves(false); + const { service, u1, app } = t.context; + app.get(ConfigFactory).override({ + payment: { + showLifetimePrice: false, + }, + }); await t.throwsAsync( () => @@ -1202,7 +1212,13 @@ test('should not be able to checkout for lifetime recurring if not enabled', asy }); test('should be able to checkout for lifetime recurring', async t => { - const { service, u1, stripe } = t.context; + const { service, u1, stripe, app } = t.context; + + app.get(ConfigFactory).override({ + payment: { + showLifetimePrice: true, + }, + }); await service.checkout( { diff --git a/packages/backend/server/src/__tests__/utils/common.ts b/packages/backend/server/src/__tests__/utils/common.ts deleted file mode 100644 index 7729223676..0000000000 --- a/packages/backend/server/src/__tests__/utils/common.ts +++ /dev/null @@ -1 +0,0 @@ -export const gql = '/graphql'; diff --git a/packages/backend/server/src/__tests__/utils/copilot.ts b/packages/backend/server/src/__tests__/utils/copilot.ts index bb6efad5b4..46cfcfbaa8 100644 --- a/packages/backend/server/src/__tests__/utils/copilot.ts +++ b/packages/backend/server/src/__tests__/utils/copilot.ts @@ -1,160 +1,11 @@ -import { randomBytes } from 'node:crypto'; - -import { - DEFAULT_DIMENSIONS, - OpenAIProvider, -} from '../../plugins/copilot/providers/openai'; -import { - CopilotCapability, - CopilotChatOptions, - CopilotEmbeddingOptions, - CopilotImageToImageProvider, - CopilotImageToTextProvider, - CopilotProviderType, - CopilotTextToEmbeddingProvider, - CopilotTextToImageProvider, - CopilotTextToTextProvider, - PromptConfig, - PromptMessage, -} from '../../plugins/copilot/types'; +import { PromptConfig, PromptMessage } from '../../plugins/copilot/providers'; import { NodeExecutorType } from '../../plugins/copilot/workflow/executor'; import { WorkflowGraph, WorkflowNodeType, WorkflowParams, } from '../../plugins/copilot/workflow/types'; -import { gql } from './common'; import { TestingApp } from './testing-app'; -import { sleep } from './utils'; - -// @ts-expect-error no error -export class MockCopilotTestProvider - extends OpenAIProvider - implements - CopilotTextToTextProvider, - CopilotTextToEmbeddingProvider, - CopilotTextToImageProvider, - CopilotImageToImageProvider, - CopilotImageToTextProvider -{ - static override readonly type = CopilotProviderType.Test; - override readonly availableModels = [ - 'test', - 'gpt-4o', - 'gpt-4o-2024-08-06', - 'fast-sdxl/image-to-image', - 'lcm-sd15-i2i', - 'clarity-upscaler', - 'imageutils/rembg', - ]; - static override readonly capabilities = [ - CopilotCapability.TextToText, - CopilotCapability.TextToEmbedding, - CopilotCapability.TextToImage, - CopilotCapability.ImageToImage, - CopilotCapability.ImageToText, - ]; - - constructor() { - super({ apiKey: '1' }); - } - - override getCapabilities(): CopilotCapability[] { - return MockCopilotTestProvider.capabilities; - } - - static override assetsConfig(_config: any) { - return true; - } - - override get type(): CopilotProviderType { - return CopilotProviderType.Test; - } - - override async isModelAvailable(model: string): Promise { - return this.availableModels.includes(model); - } - - // ====== text to text ====== - - override async generateText( - messages: PromptMessage[], - model: string = 'test', - options: CopilotChatOptions = {} - ): Promise { - this.checkParams({ messages, model, options }); - // make some time gap for history test case - await sleep(100); - return 'generate text to text'; - } - - override async *generateTextStream( - messages: PromptMessage[], - model: string = 'gpt-4o-mini', - options: CopilotChatOptions = {} - ): AsyncIterable { - this.checkParams({ messages, model, options }); - - // make some time gap for history test case - await sleep(100); - const result = 'generate text to text stream'; - for (const message of result) { - yield message; - if (options.signal?.aborted) { - break; - } - } - } - - // ====== text to embedding ====== - - override async generateEmbedding( - messages: string | string[], - model: string, - options: CopilotEmbeddingOptions = { dimensions: DEFAULT_DIMENSIONS } - ): Promise { - messages = Array.isArray(messages) ? messages : [messages]; - this.checkParams({ embeddings: messages, model, options }); - - // make some time gap for history test case - await sleep(100); - return [Array.from(randomBytes(options.dimensions)).map(v => v % 128)]; - } - - // ====== text to image ====== - override async generateImages( - messages: PromptMessage[], - model: string = 'test', - _options: { - signal?: AbortSignal; - user?: string; - } = {} - ): Promise> { - const { content: prompt } = messages[0] || {}; - if (!prompt) { - throw new Error('Prompt is required'); - } - - // make some time gap for history test case - await sleep(100); - // just let test case can easily verify the final prompt - return [`https://example.com/${model}.jpg`, prompt]; - } - - override async *generateImagesStream( - messages: PromptMessage[], - model: string = 'dall-e-3', - options: { - signal?: AbortSignal; - user?: string; - } = {} - ): AsyncIterable { - const ret = await this.generateImages(messages, model, options); - for (const url of ret) { - yield url; - } - } -} export const cleanObject = ( obj: any[] | undefined, @@ -342,7 +193,7 @@ export async function addContextFile( content: Buffer ): Promise<{ id: string }> { const res = await app - .POST(gql) + .POST('/graphql') .set({ 'x-request-id': 'test', 'x-operation-name': 'test' }) .field( 'operations', diff --git a/packages/backend/server/src/__tests__/utils/testing-app.ts b/packages/backend/server/src/__tests__/utils/testing-app.ts index d37637a852..2ed5e61041 100644 --- a/packages/backend/server/src/__tests__/utils/testing-app.ts +++ b/packages/backend/server/src/__tests__/utils/testing-app.ts @@ -41,20 +41,17 @@ export async function createTestingApp( moduleDef: TestingAppMetadata = {} ): Promise { const module = await createTestingModule(moduleDef, false); + const logger = new AFFiNELogger(); + logger.setLogLevels([TEST_LOG_LEVEL]); const app = module.createNestApplication({ cors: true, bodyParser: true, rawBody: true, + logger, }); app.useBodyParser('raw', { limit: 1 * OneMB }); - - const logger = new AFFiNELogger(); - - logger.setLogLevels([TEST_LOG_LEVEL]); - app.useLogger(logger); - app.useGlobalFilters(new GlobalExceptionFilter(app.getHttpAdapter())); app.use( graphqlUploadExpress({ diff --git a/packages/backend/server/src/__tests__/utils/testing-module.ts b/packages/backend/server/src/__tests__/utils/testing-module.ts index 68bfa858d1..c37e9589bb 100644 --- a/packages/backend/server/src/__tests__/utils/testing-module.ts +++ b/packages/backend/server/src/__tests__/utils/testing-module.ts @@ -8,9 +8,10 @@ import { } from '@nestjs/testing'; import { PrismaClient } from '@prisma/client'; -import { AppModule, FunctionalityModules } from '../../app.module'; -import { AFFiNELogger, JobQueue, Runtime } from '../../base'; +import { buildAppModule, FunctionalityModules } from '../../app.module'; +import { AFFiNELogger, JobQueue } from '../../base'; import { GqlModule } from '../../base/graphql'; +import { ServerConfigModule } from '../../core'; import { AuthGuard, AuthModule } from '../../core/auth'; import { Mailer, MailModule } from '../../core/mail'; import { ModelsModule } from '../../models'; @@ -63,16 +64,18 @@ export async function createTestingModule( autoInitialize = true ): Promise { // setting up - let imports = moduleDef.imports ?? [AppModule]; + let imports = moduleDef.imports ?? [buildAppModule(globalThis.env)]; imports = - imports[0] === AppModule - ? [AppModule] + // @ts-expect-error + imports[0].module?.name === 'AppModule' + ? imports : dedupeModules([ ...FunctionalityModules, ModelsModule, AuthModule, GqlModule, MailModule, + ServerConfigModule, ...imports, ]); @@ -101,10 +104,6 @@ export async function createTestingModule( testingModule.initTestingDB = async () => { await initTestingDB(module); - - const runtime = module.get(Runtime); - // by pass password min length validation - await runtime.set('auth/password.min', 1); }; testingModule.create = createFactory( diff --git a/packages/backend/server/src/__tests__/utils/utils.ts b/packages/backend/server/src/__tests__/utils/utils.ts index 97a15c915b..efd00699ab 100644 --- a/packages/backend/server/src/__tests__/utils/utils.ts +++ b/packages/backend/server/src/__tests__/utils/utils.ts @@ -1,6 +1,7 @@ import { INestApplicationContext, LogLevel } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; import { PrismaClient } from '@prisma/client'; +import whywhywhy from 'why-is-node-running'; import { RefreshFeatures0001 } from '../../data/migrations/0001-refresh-features'; @@ -32,3 +33,21 @@ export async function initTestingDB(context: INestApplicationContext) { export async function sleep(ms: number) { return new Promise(resolve => setTimeout(resolve, ms)); } + +export function debugProcessHolding(ignorePrismaStack = true) { + setImmediate(() => { + whywhywhy({ + error: message => { + // ignore prisma error + if ( + ignorePrismaStack && + (message.includes('Prisma') || message.includes('prisma')) + ) { + return; + } + + console.error(message); + }, + }); + }); +} diff --git a/packages/backend/server/src/__tests__/version.spec.ts b/packages/backend/server/src/__tests__/version.spec.ts index 8b54b7289e..e09778c2ed 100644 --- a/packages/backend/server/src/__tests__/version.spec.ts +++ b/packages/backend/server/src/__tests__/version.spec.ts @@ -3,7 +3,7 @@ import test from 'ava'; import Sinon from 'sinon'; import { AppModule } from '../app.module'; -import { Runtime, UseNamedGuard } from '../base'; +import { ConfigFactory, UseNamedGuard } from '../base'; import { Public } from '../core/auth/guard'; import { VersionService } from '../core/version/service'; import { createTestingApp, TestingApp } from './utils'; @@ -19,28 +19,28 @@ class GuardedController { } let app: TestingApp; -let runtime: Sinon.SinonStubbedInstance; +let config: ConfigFactory; let version: VersionService; function checkVersion(enabled = true) { - runtime.fetch.withArgs('client/versionControl.enabled').resolves(enabled); - - runtime.fetch - .withArgs('client/versionControl.requiredVersion') - .resolves('>=0.20.0'); + config.override({ + client: { + versionControl: { + enabled, + requiredVersion: '>=0.20.0', + }, + }, + }); } test.before(async () => { app = await createTestingApp({ imports: [AppModule], controllers: [GuardedController], - tapModule: m => { - m.overrideProvider(Runtime).useValue(Sinon.createStubInstance(Runtime)); - }, }); - runtime = app.get(Runtime); version = app.get(VersionService, { strict: false }); + config = app.get(ConfigFactory, { strict: false }); }); test.beforeEach(async () => { @@ -74,9 +74,13 @@ test('should passthrough if version check is not enabled', async t => { }); test('should passthrough is version range is invalid', async t => { - runtime.fetch - .withArgs('client/versionControl.requiredVersion') - .resolves('invalid'); + config.override({ + client: { + versionControl: { + requiredVersion: 'invalid', + }, + }, + }); let res = await app.GET('/guarded/test').set('x-affine-version', 'invalid'); @@ -92,9 +96,13 @@ test('should pass if client version is allowed', async t => { t.is(res.status, 200); - runtime.fetch - .withArgs('client/versionControl.requiredVersion') - .resolves('>=0.19.0'); + config.override({ + client: { + versionControl: { + requiredVersion: '>=0.19.0', + }, + }, + }); res = await app.GET('/guarded/test').set('x-affine-version', '0.19.0'); @@ -120,9 +128,13 @@ test('should fail if client version is not set or invalid', async t => { }); test('should tell upgrade if client version is lower than allowed', async t => { - runtime.fetch - .withArgs('client/versionControl.requiredVersion') - .resolves('>=0.21.0 <=0.22.0'); + config.override({ + client: { + versionControl: { + requiredVersion: '>=0.21.0 <=0.22.0', + }, + }, + }); let res = await app.GET('/guarded/test').set('x-affine-version', '0.20.0'); @@ -134,9 +146,13 @@ test('should tell upgrade if client version is lower than allowed', async t => { }); test('should tell downgrade if client version is higher than allowed', async t => { - runtime.fetch - .withArgs('client/versionControl.requiredVersion') - .resolves('>=0.20.0 <=0.22.0'); + config.override({ + client: { + versionControl: { + requiredVersion: '>=0.20.0 <=0.22.0', + }, + }, + }); let res = await app.GET('/guarded/test').set('x-affine-version', '0.23.0'); @@ -148,9 +164,13 @@ test('should tell downgrade if client version is higher than allowed', async t = }); test('should test prerelease version', async t => { - runtime.fetch - .withArgs('client/versionControl.requiredVersion') - .resolves('>=0.19.0'); + config.override({ + client: { + versionControl: { + requiredVersion: '>=0.19.0', + }, + }, + }); let res = await app .GET('/guarded/test') diff --git a/packages/backend/server/src/__tests__/worker.e2e.ts b/packages/backend/server/src/__tests__/worker.e2e.ts index 8ccfd423d1..6521ee61df 100644 --- a/packages/backend/server/src/__tests__/worker.e2e.ts +++ b/packages/backend/server/src/__tests__/worker.e2e.ts @@ -3,7 +3,6 @@ import ava from 'ava'; import Sinon from 'sinon'; import type { Response } from 'supertest'; -import { WorkerModule } from '../plugins/worker'; import { createTestingApp, TestingApp } from './utils'; type TestContext = { @@ -13,9 +12,9 @@ type TestContext = { const test = ava as TestFn; test.before(async t => { - const app = await createTestingApp({ - imports: [WorkerModule], - }); + // @ts-expect-error test + env.DEPLOYMENT_TYPE = 'selfhosted'; + const app = await createTestingApp(); t.context.app = app; }); diff --git a/packages/backend/server/src/app.controller.ts b/packages/backend/server/src/app.controller.ts index 99f5a4990a..e4a3a90f5a 100644 --- a/packages/backend/server/src/app.controller.ts +++ b/packages/backend/server/src/app.controller.ts @@ -1,21 +1,19 @@ import { Controller, Get } from '@nestjs/common'; -import { Config, SkipThrottle } from './base'; +import { SkipThrottle } from './base'; import { Public } from './core/auth'; @Controller('/info') export class AppController { - constructor(private readonly config: Config) {} - @SkipThrottle() @Public() @Get() info() { return { - compatibility: this.config.version, - message: `AFFiNE ${this.config.version} Server`, - type: this.config.type, - flavor: this.config.flavor.type, + compatibility: env.version, + message: `AFFiNE ${env.version} Server`, + type: env.DEPLOYMENT_TYPE, + flavor: env.FLAVOR, }; } } diff --git a/packages/backend/server/src/app.module.ts b/packages/backend/server/src/app.module.ts index 762a0159c7..ce32d008a7 100644 --- a/packages/backend/server/src/app.module.ts +++ b/packages/backend/server/src/app.module.ts @@ -1,27 +1,19 @@ -import { - DynamicModule, - ExecutionContext, - ForwardReference, - Logger, - Module, -} from '@nestjs/common'; +import { DynamicModule, ExecutionContext } from '@nestjs/common'; import { ScheduleModule } from '@nestjs/schedule'; import { ClsPluginTransactional } from '@nestjs-cls/transactional'; import { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-prisma'; import { PrismaClient } from '@prisma/client'; import { Request, Response } from 'express'; -import { get } from 'lodash-es'; import { ClsModule } from 'nestjs-cls'; import { AppController } from './app.controller'; import { - getOptionalModuleMetadata, getRequestIdFromHost, getRequestIdFromRequest, ScannerModule, } from './base'; import { CacheModule } from './base/cache'; -import { AFFiNEConfig, ConfigModule, mergeConfigOverride } from './base/config'; +import { ConfigModule } from './base/config'; import { ErrorModule } from './base/error'; import { EventModule } from './base/event'; import { GqlModule } from './base/graphql'; @@ -32,12 +24,11 @@ import { MetricsModule } from './base/metrics'; import { MutexModule } from './base/mutex'; import { PrismaModule } from './base/prisma'; import { RedisModule } from './base/redis'; -import { RuntimeModule } from './base/runtime'; import { StorageProviderModule } from './base/storage'; import { RateLimiterModule } from './base/throttler'; import { WebSocketModule } from './base/websocket'; import { AuthModule } from './core/auth'; -import { ADD_ENABLED_FEATURES, ServerConfigModule } from './core/config'; +import { ServerConfigModule, ServerConfigResolverModule } from './core/config'; import { DocStorageModule } from './core/doc'; import { DocRendererModule } from './core/doc-renderer'; import { DocServiceModule } from './core/doc-service'; @@ -52,10 +43,16 @@ import { SyncModule } from './core/sync'; import { UserModule } from './core/user'; import { VersionModule } from './core/version'; import { WorkspaceModule } from './core/workspaces'; +import { Env } from './env'; import { ModelsModule } from './models'; -import { REGISTERED_PLUGINS } from './plugins'; +import { CaptchaModule } from './plugins/captcha'; +import { CopilotModule } from './plugins/copilot'; +import { CustomerIoModule } from './plugins/customerio'; +import { GCloudModule } from './plugins/gcloud'; import { LicenseModule } from './plugins/license'; -import { ENABLED_PLUGINS } from './plugins/registry'; +import { OAuthModule } from './plugins/oauth'; +import { PaymentModule } from './plugins/payment'; +import { WorkerModule } from './plugins/worker'; export const FunctionalityModules = [ ClsModule.forRoot({ @@ -91,126 +88,64 @@ export const FunctionalityModules = [ }), ], }), - ConfigModule.forRoot(), - RuntimeModule, + LoggerModule, ScannerModule, + PrismaModule, EventModule, + ConfigModule, RedisModule, CacheModule, MutexModule, - PrismaModule, MetricsModule, RateLimiterModule, StorageProviderModule, HelpersModule, ErrorModule, - LoggerModule, WebSocketModule, JobModule.forRoot(), + ModelsModule, ]; -function filterOptionalModule( - config: AFFiNEConfig, - module: AFFiNEModule | Promise | ForwardReference -) { - // can't deal with promise or forward reference - if (module instanceof Promise || 'forwardRef' in module) { - return module; - } - - const requirements = getOptionalModuleMetadata(module, 'requires'); - // if condition not set or condition met, include the module - if (requirements?.length) { - const nonMetRequirements = requirements.filter(c => { - const value = get(config, c); - return ( - value === undefined || - value === null || - (typeof value === 'string' && value.trim().length === 0) - ); - }); - - if (nonMetRequirements.length) { - const name = 'module' in module ? module.module.name : module.name; - if (!config.node.test) { - new Logger(name).warn( - `${name} is not enabled because of the required configuration is not satisfied.`, - 'Unsatisfied configuration:', - ...nonMetRequirements.map(config => ` AFFiNE.${config}`) - ); - } - return null; - } - } - - const predicator = getOptionalModuleMetadata(module, 'if'); - if (predicator && !predicator(config)) { - return null; - } - - const contribution = getOptionalModuleMetadata(module, 'contributesTo'); - if (contribution) { - ADD_ENABLED_FEATURES(contribution); - } - - const subModules = getOptionalModuleMetadata(module, 'imports'); - const filteredSubModules = subModules - ?.map(subModule => filterOptionalModule(config, subModule)) - .filter(Boolean); - Reflect.defineMetadata('imports', filteredSubModules, module); - - return module; -} - export class AppModuleBuilder { private readonly modules: AFFiNEModule[] = []; - constructor(private readonly config: AFFiNEConfig) {} use(...modules: AFFiNEModule[]): this { modules.forEach(m => { - const result = filterOptionalModule(this.config, m); - if (result) { - this.modules.push(m); - } + this.modules.push(m); }); return this; } - useIf( - predicator: (config: AFFiNEConfig) => boolean, - ...modules: AFFiNEModule[] - ): this { - if (predicator(this.config)) { + useIf(predicator: () => boolean, ...modules: AFFiNEModule[]): this { + if (predicator()) { this.use(...modules); } return this; } - compile() { - @Module({ - imports: this.modules, - controllers: [AppController], - }) + compile(): DynamicModule { class AppModule {} - return AppModule; + return { + module: AppModule, + imports: this.modules, + controllers: [AppController], + }; } } -export function buildAppModule() { - AFFiNE = mergeConfigOverride(AFFiNE); - const factor = new AppModuleBuilder(AFFiNE); +export function buildAppModule(env: Env) { + const factor = new AppModuleBuilder(); factor // basic .use(...FunctionalityModules) - .use(ModelsModule) // enable schedule module on graphql server and doc service .useIf( - config => config.flavor.graphql || config.flavor.doc, + () => env.flavors.graphql || env.flavors.doc, ScheduleModule.forRoot() ) @@ -219,46 +154,41 @@ export function buildAppModule() { // business modules .use( + ServerConfigModule, FeatureModule, QuotaModule, DocStorageModule, NotificationModule, MailModule ) - + // renderer server only + .useIf(() => env.flavors.renderer, DocRendererModule) // sync server only - .useIf(config => config.flavor.sync, SyncModule) - + .useIf(() => env.flavors.sync, SyncModule) // graphql server only .useIf( - config => config.flavor.graphql, - VersionModule, + () => env.flavors.graphql, GqlModule, + VersionModule, StorageModule, - ServerConfigModule, + ServerConfigResolverModule, WorkspaceModule, - LicenseModule + LicenseModule, + PaymentModule, + CopilotModule, + CaptchaModule, + OAuthModule, + CustomerIoModule ) - // doc service only - .useIf(config => config.flavor.doc, DocServiceModule) - + .useIf(() => env.flavors.doc, DocServiceModule) // self hosted server only - .useIf(config => config.isSelfhosted, SelfhostModule) - .useIf(config => config.flavor.renderer, DocRendererModule); + .useIf(() => env.selfhosted, WorkerModule, SelfhostModule) - // plugin modules - ENABLED_PLUGINS.forEach(name => { - const plugin = REGISTERED_PLUGINS.get(name); - if (!plugin) { - new Logger('AppBuilder').warn(`Unknown plugin ${name}`); - return; - } - - factor.use(plugin); - }); + // gcloud + .useIf(() => env.gcp, GCloudModule); return factor.compile(); } -export const AppModule = buildAppModule(); +export const AppModule = buildAppModule(env); diff --git a/packages/backend/server/src/app.ts b/packages/backend/server/src/app.ts index cad0d93974..50df53b980 100644 --- a/packages/backend/server/src/app.ts +++ b/packages/backend/server/src/app.ts @@ -7,11 +7,11 @@ import { AFFiNELogger, CacheInterceptor, CloudThrottlerGuard, + Config, GlobalExceptionFilter, } from './base'; import { SocketIoAdapter } from './base/websocket'; import { AuthGuard } from './core/auth'; -import { ENABLED_FEATURES } from './core/config/server-feature'; import { serverTimingAndCache } from './middleware/timing'; const OneMB = 1024 * 1024; @@ -29,9 +29,10 @@ export async function createApp() { app.useBodyParser('raw', { limit: 100 * OneMB }); app.useLogger(app.get(AFFiNELogger)); + const config = app.get(Config); - if (AFFiNE.server.path) { - app.setGlobalPrefix(AFFiNE.server.path); + if (config.server.path) { + app.setGlobalPrefix(config.server.path); } app.use(serverTimingAndCache); @@ -49,22 +50,12 @@ export async function createApp() { app.use(cookieParser()); // only enable shutdown hooks in production // https://docs.nestjs.com/fundamentals/lifecycle-events#application-shutdown - if (AFFiNE.NODE_ENV === 'production') { + if (env.prod) { app.enableShutdownHooks(); } const adapter = new SocketIoAdapter(app); app.useWebSocketAdapter(adapter); - if (AFFiNE.isSelfhosted && AFFiNE.metrics.telemetry.enabled) { - const mixpanel = await import('mixpanel'); - mixpanel - .init(AFFiNE.metrics.telemetry.token) - .track('selfhost-server-started', { - version: AFFiNE.version, - features: Array.from(ENABLED_FEATURES), - }); - } - return app; } diff --git a/packages/backend/server/src/base/config/__tests__/config.spec.ts b/packages/backend/server/src/base/config/__tests__/config.spec.ts new file mode 100644 index 0000000000..ac0cc706a8 --- /dev/null +++ b/packages/backend/server/src/base/config/__tests__/config.spec.ts @@ -0,0 +1,90 @@ +import test from 'ava'; + +import { createModule } from '../../../__tests__/create-module'; +import { ConfigFactory, ConfigModule } from '..'; +import { Config } from '../config'; + +const module = await createModule(); +test.after.always(async () => { + await module.close(); +}); + +test('should create config', t => { + const config = module.get(Config); + + t.is(typeof config.auth.passwordRequirements.max, 'number'); + t.is(typeof config.job.queue, 'object'); +}); + +test('should override config', async t => { + await using module = await createModule({ + imports: [ + ConfigModule.override({ + auth: { + passwordRequirements: { + max: 100, + min: 6, + }, + }, + job: { + queues: { + notification: { + concurrency: 1000, + }, + }, + }, + }), + ], + }); + + const config = module.get(Config); + const configFactory = module.get(ConfigFactory); + + t.deepEqual(config.auth.passwordRequirements, { + max: 100, + min: 6, + }); + + configFactory.override({ + auth: { + passwordRequirements: { + max: 10, + }, + }, + }); + + t.deepEqual(config.auth.passwordRequirements, { + max: 10, + min: 6, + }); +}); + +test('should validate config', t => { + const config = module.get(ConfigFactory); + + t.notThrows(() => + config.validate([ + { + module: 'auth', + key: 'passwordRequirements', + value: { max: 10, min: 6 }, + }, + ]) + ); + + t.throws( + () => + config.validate([ + { + module: 'auth', + key: 'passwordRequirements', + value: { max: 10, min: 10 }, + }, + ]), + { + message: `Invalid config for module [auth] with key [passwordRequirements] +Value: {"max":10,"min":10} +Error: Minimum length of password must be less than maximum length`, + } + ); +}); diff --git a/packages/backend/server/src/base/config/config.ts b/packages/backend/server/src/base/config/config.ts new file mode 100644 index 0000000000..1f5e1bc185 --- /dev/null +++ b/packages/backend/server/src/base/config/config.ts @@ -0,0 +1,3 @@ +import { ApplyType } from '../utils'; + +export class Config extends ApplyType() {} diff --git a/packages/backend/server/src/base/config/def.ts b/packages/backend/server/src/base/config/def.ts deleted file mode 100644 index f25676c982..0000000000 --- a/packages/backend/server/src/base/config/def.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { LeafPaths } from '../utils/types'; -import { AppStartupConfig } from './types'; - -export type EnvConfigType = 'string' | 'int' | 'float' | 'boolean'; -export type ServerFlavor = - | 'allinone' - | 'graphql' - | 'sync' - | 'renderer' - | 'doc' - | 'script'; - -export type AFFINE_ENV = 'dev' | 'beta' | 'production'; -export type NODE_ENV = 'development' | 'test' | 'production'; - -export enum DeploymentType { - Affine = 'affine', - Selfhosted = 'selfhosted', -} - -export type ConfigPaths = LeafPaths; - -export interface PreDefinedAFFiNEConfig { - ENV_MAP: Record; - serverId: string; - serverName: string; - readonly projectRoot: string; - readonly AFFINE_ENV: AFFINE_ENV; - readonly NODE_ENV: NODE_ENV; - readonly version: string; - readonly type: DeploymentType; - readonly isSelfhosted: boolean; - readonly flavor: { type: string } & { [key in ServerFlavor]: boolean }; - readonly affine: { canary: boolean; beta: boolean; stable: boolean }; - readonly node: { - prod: boolean; - dev: boolean; - test: boolean; - }; - readonly deploy: boolean; -} - -export interface AppPluginsConfig {} - -export type AFFiNEConfig = PreDefinedAFFiNEConfig & - AppStartupConfig & - AppPluginsConfig; - -declare global { - // oxlint-disable-next-line @typescript-eslint/no-namespace - namespace globalThis { - // oxlint-disable-next-line no-var - var AFFiNE: AFFiNEConfig; - } -} diff --git a/packages/backend/server/src/base/config/default.ts b/packages/backend/server/src/base/config/default.ts deleted file mode 100644 index 51b1092bb6..0000000000 --- a/packages/backend/server/src/base/config/default.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import pkg from '../../../package.json' with { type: 'json' }; -import { - AFFINE_ENV, - AFFiNEConfig, - DeploymentType, - NODE_ENV, - PreDefinedAFFiNEConfig, - ServerFlavor, -} from './def'; -import { readEnv } from './env'; -import { defaultStartupConfig } from './register'; - -function expectFlavor(flavor: ServerFlavor, expected: ServerFlavor) { - return flavor === expected || flavor === 'allinone'; -} - -function getPredefinedAFFiNEConfig(): PreDefinedAFFiNEConfig { - const NODE_ENV = readEnv('NODE_ENV', 'production', [ - 'development', - 'test', - 'production', - ]); - const AFFINE_ENV = readEnv('AFFINE_ENV', 'production', [ - 'dev', - 'beta', - 'production', - ]); - const flavor = readEnv('SERVER_FLAVOR', 'allinone', [ - 'allinone', - 'graphql', - 'sync', - 'renderer', - 'doc', - 'script', - ]); - const deploymentType = readEnv( - 'DEPLOYMENT_TYPE', - NODE_ENV === 'development' - ? DeploymentType.Affine - : DeploymentType.Selfhosted, - Object.values(DeploymentType) - ); - const isSelfhosted = deploymentType === DeploymentType.Selfhosted; - const affine = { - canary: AFFINE_ENV === 'dev', - beta: AFFINE_ENV === 'beta', - stable: AFFINE_ENV === 'production', - }; - const node = { - prod: NODE_ENV === 'production', - dev: NODE_ENV === 'development', - test: NODE_ENV === 'test', - }; - - return { - ENV_MAP: {}, - NODE_ENV, - AFFINE_ENV, - serverId: 'some-randome-uuid', - serverName: isSelfhosted ? 'Self-Host Cloud' : 'AFFiNE Cloud', - version: pkg.version, - type: deploymentType, - isSelfhosted, - flavor: { - type: flavor, - allinone: flavor === 'allinone', - graphql: expectFlavor(flavor, 'graphql'), - sync: expectFlavor(flavor, 'sync'), - renderer: expectFlavor(flavor, 'renderer'), - doc: expectFlavor(flavor, 'doc'), - script: expectFlavor(flavor, 'script'), - }, - affine, - node, - deploy: !node.dev && !node.test, - projectRoot: resolve(fileURLToPath(import.meta.url), '../../../../'), - }; -} - -export function getAFFiNEConfigModifier(): AFFiNEConfig { - const predefined = getPredefinedAFFiNEConfig() as AFFiNEConfig; - - return chainableProxy(predefined); -} - -function merge(a: any, b: any) { - if (typeof b !== 'object' || b instanceof Map || b instanceof Set) { - return b; - } - - if (Array.isArray(b)) { - if (Array.isArray(a)) { - return a.concat(b); - } - return b; - } - - const result = { ...a }; - Object.keys(b).forEach(key => { - result[key] = merge(result[key], b[key]); - }); - - return result; -} - -export function mergeConfigOverride(override: any) { - return merge(defaultStartupConfig, override); -} - -function chainableProxy(obj: any) { - const keys: Set = new Set(Object.keys(obj)); - return new Proxy(obj, { - get(target, prop) { - if (!(prop in target)) { - keys.add(prop as string); - target[prop] = chainableProxy({}); - } - return target[prop]; - }, - set(target, prop, value) { - keys.add(prop as string); - if ( - typeof value === 'object' && - !( - value instanceof Map || - value instanceof Set || - value instanceof Array - ) - ) { - value = chainableProxy(value); - } - target[prop] = value; - return true; - }, - ownKeys() { - return Array.from(keys); - }, - }); -} diff --git a/packages/backend/server/src/base/config/env.ts b/packages/backend/server/src/base/config/env.ts index b05065f4cd..994d9c8bc9 100644 --- a/packages/backend/server/src/base/config/env.ts +++ b/packages/backend/server/src/base/config/env.ts @@ -1,11 +1,9 @@ -import { set } from 'lodash-es'; - -import type { AFFiNEConfig, EnvConfigType } from './def'; +export type EnvConfigType = 'string' | 'integer' | 'float' | 'boolean'; /** * parse number value from environment variables */ -function int(value: string) { +function integer(value: string) { const n = parseInt(value); return Number.isNaN(n) ? undefined : n; } @@ -20,7 +18,7 @@ function boolean(value: string) { } const envParsers: Record unknown> = { - int, + integer, float, boolean, string: value => value, @@ -33,38 +31,3 @@ export function parseEnvValue(value: string | undefined, type: EnvConfigType) { return envParsers[type](value); } - -export function applyEnvToConfig(rawConfig: AFFiNEConfig) { - for (const env in rawConfig.ENV_MAP) { - const config = rawConfig.ENV_MAP[env]; - const [path, value] = - typeof config === 'string' - ? [config, parseEnvValue(process.env[env], 'string')] - : [config[0], parseEnvValue(process.env[env], config[1] ?? 'string')]; - - if (value !== undefined) { - set(rawConfig, path, value); - } - } -} - -export function readEnv( - env: string, - defaultValue: T, - availableValues?: T[] -) { - const value = process.env[env]; - if (value === undefined) { - return defaultValue; - } - - if (availableValues && !availableValues.includes(value as any)) { - throw new Error( - `Invalid value '${value}' for environment variable ${env}, expected one of [${availableValues.join( - ', ' - )}]` - ); - } - - return value as T; -} diff --git a/packages/backend/server/src/base/config/factory.ts b/packages/backend/server/src/base/config/factory.ts new file mode 100644 index 0000000000..ad6c7a8877 --- /dev/null +++ b/packages/backend/server/src/base/config/factory.ts @@ -0,0 +1,60 @@ +import { Inject, Injectable, Optional } from '@nestjs/common'; +import { merge } from 'lodash-es'; + +import { InvalidAppConfig } from '../error'; +import { APP_CONFIG_DESCRIPTORS, getDefaultConfig } from './register'; + +export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN'); + +@Injectable() +export class ConfigFactory { + readonly #config: DeepReadonly; + + constructor( + @Inject(OVERRIDE_CONFIG_TOKEN) + @Optional() + private readonly overrides: DeepPartial = {} + ) { + this.#config = this.loadDefault(); + } + + get config() { + return this.#config; + } + + override(updates: DeepPartial) { + merge(this.#config, updates); + } + + validate(updates: Array<{ module: string; key: string; value: any }>) { + const errors: string[] = []; + + updates.forEach(update => { + const descriptor = APP_CONFIG_DESCRIPTORS[update.module]?.[update.key]; + if (!descriptor) { + errors.push( + `Invalid config for module [${update.module}] with unknown key [${update.key}]` + ); + return; + } + + const { success, error } = descriptor.validate(update.value); + if (!success) { + error.issues.forEach(issue => { + errors.push(`Invalid config for module [${update.module}] with key [${update.key}] +Value: ${JSON.stringify(update.value)} +Error: ${issue.message}`); + }); + } + }); + + if (errors.length > 0) { + throw new InvalidAppConfig(errors.join('\n')); + } + } + + private loadDefault(): DeepReadonly { + const config = getDefaultConfig(); + return merge(config, this.overrides); + } +} diff --git a/packages/backend/server/src/base/config/index.ts b/packages/backend/server/src/base/config/index.ts index 7bbe2d65a8..38f347981c 100644 --- a/packages/backend/server/src/base/config/index.ts +++ b/packages/backend/server/src/base/config/index.ts @@ -1,37 +1,29 @@ -import { DynamicModule, FactoryProvider } from '@nestjs/common'; -import { merge } from 'lodash-es'; +import { DynamicModule, Global, Module, Provider } from '@nestjs/common'; -import { AFFiNEConfig } from './def'; -import { Config } from './provider'; - -export * from './def'; -export * from './default'; -export { applyEnvToConfig, parseEnvValue } from './env'; -export * from './provider'; -export { defineRuntimeConfig, defineStartupConfig } from './register'; -export type { AppConfig, ConfigItem, ModuleConfig } from './types'; - -function createConfigProvider( - override?: DeepPartial -): FactoryProvider { - return { - provide: Config, - useFactory: () => { - return Object.freeze(merge({}, globalThis.AFFiNE, override)); - }, - inject: [], - }; -} +import { Config } from './config'; +import { ConfigFactory, OVERRIDE_CONFIG_TOKEN } from './factory'; +import { ConfigProvider } from './provider'; +@Global() +@Module({ + providers: [ConfigProvider, ConfigFactory], + exports: [ConfigProvider, ConfigFactory], +}) export class ConfigModule { - static forRoot = (override?: DeepPartial): DynamicModule => { - const provider = createConfigProvider(override); + static override(overrides: DeepPartial = {}): DynamicModule { + const provider: Provider = { + provide: OVERRIDE_CONFIG_TOKEN, + useValue: overrides, + }; return { global: true, - module: ConfigModule, + module: class ConfigOverrideModule {}, providers: [provider], exports: [provider], }; - }; + } } + +export { Config, ConfigFactory }; +export { defineModuleConfig, type JSONSchema } from './register'; diff --git a/packages/backend/server/src/base/config/provider.ts b/packages/backend/server/src/base/config/provider.ts index 5e16524d11..2d6f7c925c 100644 --- a/packages/backend/server/src/base/config/provider.ts +++ b/packages/backend/server/src/base/config/provider.ts @@ -1,16 +1,12 @@ -import { ApplyType } from '../utils/types'; -import { AFFiNEConfig } from './def'; +import { FactoryProvider } from '@nestjs/common'; -/** - * @example - * - * import { Config } from '@affine/server' - * - * class TestConfig { - * constructor(private readonly config: Config) {} - * test() { - * return this.config.env - * } - * } - */ -export class Config extends ApplyType() {} +import { Config } from './config'; +import { ConfigFactory } from './factory'; + +export const ConfigProvider: FactoryProvider = { + provide: Config, + useFactory: (factory: ConfigFactory) => { + return factory.config; + }, + inject: [ConfigFactory], +}; diff --git a/packages/backend/server/src/base/config/register.ts b/packages/backend/server/src/base/config/register.ts index 9bece01065..11880b4ea7 100644 --- a/packages/backend/server/src/base/config/register.ts +++ b/packages/backend/server/src/base/config/register.ts @@ -1,66 +1,239 @@ -import { Prisma, RuntimeConfigType } from '@prisma/client'; -import { get, merge, set } from 'lodash-es'; +import { once, set } from 'lodash-es'; +import { z } from 'zod'; -import { - AppModulesConfigDef, - AppStartupConfig, - ModuleRuntimeConfigDescriptions, - ModuleStartupConfigDescriptions, -} from './types'; +import { type EnvConfigType, parseEnvValue } from './env'; +import { AppConfigByPath } from './types'; -export const defaultStartupConfig: AppStartupConfig = {} as any; -export const defaultRuntimeConfig: Record< - string, - Prisma.RuntimeConfigCreateInput -> = {} as any; +export type JSONSchema = { description?: string } & ( + | { type?: undefined; oneOf?: JSONSchema[] } + | { + type: 'string' | 'number' | 'boolean'; + enum?: string[]; + } + | { + type: 'array'; + items?: JSONSchema; + } + | { + type: 'object'; + properties?: Record; + } +); -export function runtimeConfigType(val: any): RuntimeConfigType { - if (Array.isArray(val)) { - return RuntimeConfigType.Array; - } +type ConfigType = EnvConfigType | 'array' | 'object' | 'any'; +export type ConfigDescriptor = { + desc: string; + type: ConfigType; + validate: (value: T) => z.SafeParseReturnType; + schema: JSONSchema; + default: T; + env?: [string, EnvConfigType]; + link?: string; +}; - switch (typeof val) { - case 'string': - return RuntimeConfigType.String; - case 'number': - return RuntimeConfigType.Number; - case 'boolean': - return RuntimeConfigType.Boolean; +type ConfigDefineDescriptor = { + desc: string; + default: T; + validate?: (value: T) => boolean; + shape?: z.ZodType; + env?: string | [string, EnvConfigType]; + link?: string; + schema?: JSONSchema; +}; + +function typeFromShape(shape: z.ZodType): ConfigType { + switch (shape.constructor) { + case z.ZodString: + return 'string'; + case z.ZodNumber: + return 'float'; + case z.ZodBoolean: + return 'boolean'; + case z.ZodArray: + return 'array'; + case z.ZodObject: + return 'object'; default: - return RuntimeConfigType.Object; + return 'any'; } } -function registerRuntimeConfig( - module: T, - configs: ModuleRuntimeConfigDescriptions -) { - Object.entries(configs).forEach(([key, value]) => { - defaultRuntimeConfig[`${module}/${key}`] = { - id: `${module}/${key}`, +function shapeFromType(type: ConfigType): z.ZodType { + switch (type) { + case 'string': + return z.string(); + case 'float': + return z.number(); + case 'boolean': + return z.boolean(); + case 'integer': + return z.number().int(); + case 'array': + return z.array(z.any()); + case 'object': + return z.object({}); + default: + return z.any(); + } +} + +function typeFromSchema(schema: JSONSchema): ConfigType { + if ('type' in schema) { + switch (schema.type) { + case 'string': + return 'string'; + case 'number': + return 'float'; + case 'boolean': + return 'boolean'; + case 'array': + return 'array'; + case 'object': + return 'object'; + } + } + + return 'any'; +} + +function schemaFromType(type: ConfigType): JSONSchema['type'] { + switch (type) { + case 'any': + return undefined; + case 'float': + case 'integer': + return 'number'; + default: + return type; + } +} + +function typeFromDefault(defaultValue: T): ConfigType { + if (Array.isArray(defaultValue)) { + return 'array'; + } + + switch (typeof defaultValue) { + case 'string': + return 'string'; + case 'number': + return 'float'; + case 'boolean': + return 'boolean'; + case 'object': + return 'object'; + default: + return 'any'; + } +} + +function standardizeDescriptor( + desc: ConfigDefineDescriptor +): ConfigDescriptor { + const env = desc.env + ? Array.isArray(desc.env) + ? desc.env + : ([desc.env, 'string'] as [string, EnvConfigType]) + : undefined; + + let type: ConfigType = 'any'; + + if (desc.default !== undefined && desc.default !== null) { + type = typeFromDefault(desc.default); + } else if (env) { + type = env[1]; + } else if (desc.shape) { + type = typeFromShape(desc.shape); + } else if (desc.schema) { + type = typeFromSchema(desc.schema); + } + + const shape = desc.shape ?? shapeFromType(type); + + return { + desc: desc.desc, + default: desc.default, + type, + validate: (value: T) => { + return shape.safeParse(value); + }, + env, + link: desc.link, + schema: { + type: schemaFromType(type), + description: desc.desc, + ...desc.schema, + }, + }; +} + +type ModuleConfigDescriptors = { + [K in keyof T]: ConfigDefineDescriptor; +}; + +export const APP_CONFIG_DESCRIPTORS: Record< + string, + Record> +> = {}; + +export const getDescriptors = once(() => { + return Object.entries(APP_CONFIG_DESCRIPTORS).map( + ([module, descriptors]) => ({ module, - key, - description: value.desc, - value: value.default, - type: runtimeConfigType(value.default), - }; - }); -} - -export function defineStartupConfig( - module: T, - configs: ModuleStartupConfigDescriptions -) { - set( - defaultStartupConfig, - module, - merge(get(defaultStartupConfig, module, {}), configs) + descriptors: Object.entries(descriptors).map(([key, descriptor]) => ({ + key, + descriptor, + })), + }) ); +}); + +export function defineModuleConfig( + module: T, + defs: ModuleConfigDescriptors> +) { + const descriptors: Record> = {}; + Object.entries(defs).forEach(([key, desc]) => { + descriptors[key] = standardizeDescriptor( + desc as ConfigDefineDescriptor + ); + }); + + APP_CONFIG_DESCRIPTORS[module] = { + ...APP_CONFIG_DESCRIPTORS[module], + ...descriptors, + }; } -export function defineRuntimeConfig( - module: T, - configs: ModuleRuntimeConfigDescriptions -) { - registerRuntimeConfig(module, configs); +export function getDefaultConfig(): AppConfigSchema { + const config: Record = {}; + const envs = process.env; + + for (const [module, defs] of Object.entries(APP_CONFIG_DESCRIPTORS)) { + const modulizedConfig = {}; + + for (const [key, desc] of Object.entries(defs)) { + let defaultValue = desc.default; + + if (desc.env) { + const [env, parser] = desc.env; + const envValue = envs[env]; + if (envValue) { + defaultValue = parseEnvValue(envValue, parser); + } + } + + const { success, error } = desc.validate(defaultValue); + + if (!success) { + throw error; + } + + set(modulizedConfig, key, defaultValue); + } + + config[module] = modulizedConfig; + } + + return config as AppConfigSchema; } diff --git a/packages/backend/server/src/base/config/types.ts b/packages/backend/server/src/base/config/types.ts index 0ed543cc8e..d0e3139ad8 100644 --- a/packages/backend/server/src/base/config/types.ts +++ b/packages/backend/server/src/base/config/types.ts @@ -1,127 +1,20 @@ -import { Join, PathType } from '../utils/types'; +import { LeafPaths, PathType } from '../utils'; -export type ConfigItem = T & { __type: 'ConfigItem' }; - -type ConfigDef = Record | never; - -export interface ModuleConfig< - Startup extends ConfigDef = never, - Runtime extends ConfigDef = never, -> { - startup: Startup; - runtime: Runtime; +declare global { + type ConfigItem = Leaf; + interface AppConfigSchema {} + type AppConfig = DeeplyEraseLeaf; } -export type RuntimeConfigDescription = { - desc: string; - default: T; -}; - -type ConfigItemLeaves = - T extends Record +export type AppConfigByPath = + AppConfigSchema[Module] extends infer Config ? { - [K in keyof T]: K extends string - ? T[K] extends { __type: 'ConfigItem' } - ? K - : T[K] extends PrimitiveType - ? K - : Join> - : never; - }[keyof T] - : never; - -type StartupConfigDescriptions = { - [K in keyof T]: T[K] extends Record - ? T[K] extends ConfigItem - ? V - : T[K] - : T[K]; -}; - -type ModuleConfigLeaves = - T extends Record - ? { - [K in keyof T]: K extends string - ? T[K] extends ModuleConfig - ? K - : Join> - : never; - }[keyof T] - : never; - -type FlattenModuleConfigs> = { - // @ts-expect-error allow - [K in ModuleConfigLeaves]: PathType; -}; - -type _AppStartupConfig> = { - [K in keyof T]: T[K] extends ModuleConfig - ? S - : _AppStartupConfig; -}; - -// for extending -export interface AppConfig {} -export type AppModulesConfigDef = FlattenModuleConfigs; -export type AppConfigModules = keyof AppModulesConfigDef; -export type AppStartupConfig = _AppStartupConfig; - -// app runtime config keyed by module names -export type AppRuntimeConfigByModules = { - [Module in keyof AppModulesConfigDef]: AppModulesConfigDef[Module] extends ModuleConfig< - any, - infer Runtime - > - ? Runtime extends never - ? never - : { - // @ts-expect-error allow - [K in ConfigItemLeaves]: PathType< - Runtime, - K - > extends infer Config - ? Config extends ConfigItem + [Path in LeafPaths]: Path extends string + ? PathType extends infer Item + ? Item extends Leaf ? V - : Config - : never; - } + : Item + : never + : never; + } : never; -}; - -// names of modules that have runtime config -export type AppRuntimeConfigModules = { - [Module in keyof AppRuntimeConfigByModules]: AppRuntimeConfigByModules[Module] extends never - ? never - : Module; -}[keyof AppRuntimeConfigByModules]; - -// runtime config keyed by module names flattened into config names -// { auth: { allowSignup: boolean } } => { 'auth/allowSignup': boolean } -export type FlattenedAppRuntimeConfig = UnionToIntersection< - { - [Module in keyof AppRuntimeConfigByModules]: AppModulesConfigDef[Module] extends never - ? never - : { - [K in keyof AppRuntimeConfigByModules[Module] as K extends string - ? `${Module}/${K}` - : never]: AppRuntimeConfigByModules[Module][K]; - }; - }[keyof AppRuntimeConfigByModules] ->; - -export type ModuleStartupConfigDescriptions> = - T extends ModuleConfig - ? S extends never - ? undefined - : StartupConfigDescriptions - : never; - -export type ModuleRuntimeConfigDescriptions< - Module extends keyof AppRuntimeConfigByModules, -> = AppModulesConfigDef[Module] extends never - ? never - : { - [K in keyof AppRuntimeConfigByModules[Module]]: RuntimeConfigDescription< - AppRuntimeConfigByModules[Module][K] - >; - }; diff --git a/packages/backend/server/src/base/error/def.ts b/packages/backend/server/src/base/error/def.ts index 6ddffeef15..0f81a823f5 100644 --- a/packages/backend/server/src/base/error/def.ts +++ b/packages/backend/server/src/base/error/def.ts @@ -815,4 +815,10 @@ export const USER_FRIENDLY_ERRORS = { type: 'action_forbidden', message: 'You can not mention yourself.', }, + + // app config + invalid_app_config: { + type: 'invalid_input', + message: 'Invalid app config.', + }, } satisfies Record; diff --git a/packages/backend/server/src/base/error/errors.gen.ts b/packages/backend/server/src/base/error/errors.gen.ts index 4ac1cf847f..eb8aa49662 100644 --- a/packages/backend/server/src/base/error/errors.gen.ts +++ b/packages/backend/server/src/base/error/errors.gen.ts @@ -917,6 +917,12 @@ export class MentionUserOneselfDenied extends UserFriendlyError { super('action_forbidden', 'mention_user_oneself_denied', message); } } + +export class InvalidAppConfig extends UserFriendlyError { + constructor(message?: string) { + super('invalid_input', 'invalid_app_config', message); + } +} export enum ErrorNames { INTERNAL_SERVER_ERROR, NETWORK_ERROR, @@ -1034,7 +1040,8 @@ export enum ErrorNames { UNSUPPORTED_CLIENT_VERSION, NOTIFICATION_NOT_FOUND, MENTION_USER_DOC_ACCESS_DENIED, - MENTION_USER_ONESELF_DENIED + MENTION_USER_ONESELF_DENIED, + INVALID_APP_CONFIG } registerEnumType(ErrorNames, { name: 'ErrorNames' diff --git a/packages/backend/server/src/base/error/index.ts b/packages/backend/server/src/base/error/index.ts index eb62f90ae8..f0148272c5 100644 --- a/packages/backend/server/src/base/error/index.ts +++ b/packages/backend/server/src/base/error/index.ts @@ -5,7 +5,6 @@ import { fileURLToPath } from 'node:url'; import { Logger, Module, OnModuleInit } from '@nestjs/common'; import { Args, Query, Resolver } from '@nestjs/graphql'; -import { Config } from '../config/provider'; import { generateUserFriendlyErrors } from './def'; import { ActionForbidden, ErrorDataUnionType, ErrorNames } from './errors.gen'; @@ -23,9 +22,8 @@ class ErrorResolver { }) export class ErrorModule implements OnModuleInit { logger = new Logger('ErrorModule'); - constructor(private readonly config: Config) {} onModuleInit() { - if (!this.config.node.dev) { + if (!env.dev) { return; } this.logger.log('Generating UserFriendlyError classes'); diff --git a/packages/backend/server/src/base/event/def.ts b/packages/backend/server/src/base/event/def.ts index aa4b192621..61c5b3d8ee 100644 --- a/packages/backend/server/src/base/event/def.ts +++ b/packages/backend/server/src/base/event/def.ts @@ -1,6 +1,6 @@ import { OnOptions } from 'eventemitter2'; -import { PushMetadata, sliceMetadata } from '../nestjs'; +import { PushMetadata, sliceMetadata } from '../nestjs/decorator'; declare global { /** diff --git a/packages/backend/server/src/base/event/index.ts b/packages/backend/server/src/base/event/index.ts index 1734079e45..eef2a9ffa5 100644 --- a/packages/backend/server/src/base/event/index.ts +++ b/packages/backend/server/src/base/event/index.ts @@ -6,7 +6,10 @@ import { EventHandlerScanner } from './scanner'; const EmitProvider = { provide: EventEmitter2, - useFactory: () => new EventEmitter2(), + useFactory: () => + new EventEmitter2({ + maxListeners: 100, + }), }; @Global() diff --git a/packages/backend/server/src/base/event/scanner.ts b/packages/backend/server/src/base/event/scanner.ts index ac9f251279..c2af4b288d 100644 --- a/packages/backend/server/src/base/event/scanner.ts +++ b/packages/backend/server/src/base/event/scanner.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { once } from 'lodash-es'; -import { ModuleScanner } from '../nestjs'; +import { ModuleScanner } from '../nestjs/scanner'; import { type EventName, type EventOptions, diff --git a/packages/backend/server/src/base/graphql/config.ts b/packages/backend/server/src/base/graphql/config.ts index 8dc156b178..1406322c0a 100644 --- a/packages/backend/server/src/base/graphql/config.ts +++ b/packages/backend/server/src/base/graphql/config.ts @@ -1,17 +1,27 @@ import { ApolloDriverConfig } from '@nestjs/apollo'; -import { defineStartupConfig, ModuleConfig } from '../../base/config'; +import { defineModuleConfig } from '../config'; -declare module '../../base/config' { - interface AppConfig { - graphql: ModuleConfig; +declare global { + interface AppConfigSchema { + graphql: { + apolloDriverConfig: ConfigItem; + }; } } -defineStartupConfig('graphql', { - buildSchemaOptions: { - numberScalarMode: 'integer', +defineModuleConfig('graphql', { + apolloDriverConfig: { + desc: 'The config for underlying nestjs GraphQL and apollo driver engine.', + default: { + buildSchemaOptions: { + numberScalarMode: 'integer', + }, + useGlobalPrefix: true, + playground: true, + introspection: true, + sortSchema: true, + }, + link: 'https://docs.nestjs.com/graphql/quick-start', }, - introspection: true, - playground: true, }); diff --git a/packages/backend/server/src/base/graphql/index.ts b/packages/backend/server/src/base/graphql/index.ts index ae89b41646..8338638a25 100644 --- a/packages/backend/server/src/base/graphql/index.ts +++ b/packages/backend/server/src/base/graphql/index.ts @@ -1,13 +1,12 @@ import './config'; import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; import type { ApolloDriverConfig } from '@nestjs/apollo'; import { ApolloDriver } from '@nestjs/apollo'; import { Global, Module } from '@nestjs/common'; import { GraphQLModule } from '@nestjs/graphql'; -import { Request, Response } from 'express'; +import type { Request, Response } from 'express'; import { Config } from '../config'; import { mapAnyError } from '../nestjs/exception'; @@ -26,18 +25,17 @@ export type GraphqlContext = { driver: ApolloDriver, useFactory: (config: Config) => { return { - ...config.graphql, - path: `${config.server.path}/graphql`, + ...config.graphql.apolloDriverConfig, + autoSchemaFile: join( + env.projectRoot, + env.testing + ? './node_modules/.cache/schema.gql' + : './src/schema.gql' + ), + path: '/graphql', csrfPrevention: { requestHeaders: ['content-type'], }, - autoSchemaFile: join( - fileURLToPath(import.meta.url), - config.node.dev - ? '../../../schema.gql' - : '../../../../node_modules/.cache/schema.gql' - ), - sortSchema: true, context: ({ req, res, @@ -55,7 +53,7 @@ export type GraphqlContext = { // @ts-expect-error allow assign formattedError.extensions = ufe.toJSON(); - if (config.affine.canary) { + if (env.namespaces.canary) { formattedError.extensions.stacktrace = ufe.stacktrace; } return formattedError; diff --git a/packages/backend/server/src/base/graphql/register.ts b/packages/backend/server/src/base/graphql/register.ts index 0e5e283960..592f277ded 100644 --- a/packages/backend/server/src/base/graphql/register.ts +++ b/packages/backend/server/src/base/graphql/register.ts @@ -1,4 +1,3 @@ -import { Type } from '@nestjs/common'; import { Field, FieldOptions, ObjectType } from '@nestjs/graphql'; import { ApplyType } from '../utils/types'; @@ -7,7 +6,7 @@ export function registerObjectType( fields: Record< string, { - type: () => Type; + type: () => any; options?: FieldOptions; } >, diff --git a/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts b/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts index f7585003e1..f29f76f73c 100644 --- a/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts +++ b/packages/backend/server/src/base/helpers/__tests__/crypto.spec.ts @@ -1,5 +1,3 @@ -import { createPrivateKey, createPublicKey } from 'node:crypto'; - import ava, { TestFn } from 'ava'; import Sinon from 'sinon'; @@ -9,42 +7,19 @@ const test = ava as TestFn<{ crypto: CryptoHelper; }>; -const key = `-----BEGIN EC PRIVATE KEY----- -MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49 -AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI -3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg== ------END EC PRIVATE KEY-----`; -const privateKey = createPrivateKey({ - key, - format: 'pem', - type: 'sec1', -}) - .export({ - type: 'pkcs8', - format: 'pem', - }) - .toString('utf8'); - -const publicKey = createPublicKey({ - key, - format: 'pem', - type: 'spki', -}) - .export({ - format: 'pem', - type: 'spki', - }) - .toString('utf8'); +const privateKey = `-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgS3IAkshQuSmFWGpe +rGTg2vwaC3LdcvBQlYHHMBYJZMyhRANCAAQXdT/TAh4neNEpd4UqpDIEqWv0XvFo +BRJxGsC5I/fetqObdx1+KEjcm8zFU2xLaUTw9IZCu8OslloOjQv4ur0a +-----END PRIVATE KEY-----`; test.beforeEach(async t => { t.context.crypto = new CryptoHelper({ crypto: { - secret: { - publicKey, - privateKey, - }, + privateKey, }, } as any); + t.context.crypto.onConfigInit(); }); test('should be able to sign and verify', t => { diff --git a/packages/backend/server/src/base/helpers/config.ts b/packages/backend/server/src/base/helpers/config.ts index 481df28593..23608e1f18 100644 --- a/packages/backend/server/src/base/helpers/config.ts +++ b/packages/backend/server/src/base/helpers/config.ts @@ -1,53 +1,18 @@ -import { createPrivateKey, createPublicKey } from 'node:crypto'; +import { defineModuleConfig } from '../config'; -import { defineStartupConfig, ModuleConfig } from '../config'; - -declare module '../config' { - interface AppConfig { - crypto: ModuleConfig<{ - secret: { - publicKey: string; - privateKey: string; - }; - }>; +declare global { + interface AppConfigSchema { + crypto: { + privateKey: string; + }; } } -// Don't use this in production -const examplePrivateKey = `-----BEGIN EC PRIVATE KEY----- -MHcCAQEEIEtyAJLIULkphVhqXqxk4Nr8Ggty3XLwUJWBxzAWCWTMoAoGCCqGSM49 -AwEHoUQDQgAEF3U/0wIeJ3jRKXeFKqQyBKlr9F7xaAUScRrAuSP33rajm3cdfihI -3JvMxVNsS2lE8PSGQrvDrJZaDo0L+Lq9Gg== ------END EC PRIVATE KEY-----`; - -defineStartupConfig('crypto', { - secret: (function () { - const AFFINE_PRIVATE_KEY = - process.env.AFFINE_PRIVATE_KEY ?? examplePrivateKey; - const privateKey = createPrivateKey({ - key: Buffer.from(AFFINE_PRIVATE_KEY), - format: 'pem', - type: 'sec1', - }) - .export({ - format: 'pem', - type: 'pkcs8', - }) - .toString('utf8'); - const publicKey = createPublicKey({ - key: Buffer.from(AFFINE_PRIVATE_KEY), - format: 'pem', - type: 'spki', - }) - .export({ - format: 'pem', - type: 'spki', - }) - .toString('utf8'); - - return { - publicKey, - privateKey, - }; - })(), +defineModuleConfig('crypto', { + privateKey: { + desc: 'The private key for used by the crypto module to create signed tokens or encrypt data.', + env: 'AFFINE_PRIVATE_KEY', + default: '', + schema: { type: 'string' }, + }, }); diff --git a/packages/backend/server/src/base/helpers/crypto.ts b/packages/backend/server/src/base/helpers/crypto.ts index ee57fb505c..7767f1c36d 100644 --- a/packages/backend/server/src/base/helpers/crypto.ts +++ b/packages/backend/server/src/base/helpers/crypto.ts @@ -2,8 +2,11 @@ import { createCipheriv, createDecipheriv, createHash, + createPrivateKey, + createPublicKey, createSign, createVerify, + generateKeyPairSync, randomBytes, randomInt, timingSafeEqual, @@ -16,13 +19,48 @@ import { } from '@node-rs/argon2'; import { Config } from '../config'; +import { OnEvent } from '../event'; const NONCE_LENGTH = 12; const AUTH_TAG_LENGTH = 12; +function generatePrivateKey(): string { + const { privateKey } = generateKeyPairSync('ec', { + namedCurve: 'prime256v1', + }); + + const key = privateKey.export({ + type: 'sec1', + format: 'pem', + }); + + return key.toString('utf8'); +} + +function readPrivateKey(privateKey: string) { + return createPrivateKey({ + key: Buffer.from(privateKey), + format: 'pem', + type: 'sec1', + }) + .export({ + format: 'pem', + type: 'pkcs8', + }) + .toString('utf8'); +} + +function readPublicKey(privateKey: string) { + return createPublicKey({ + key: Buffer.from(privateKey), + }) + .export({ format: 'pem', type: 'spki' }) + .toString('utf8'); +} + @Injectable() export class CryptoHelper { - keyPair: { + keyPair!: { publicKey: Buffer; privateKey: Buffer; sha256: { @@ -31,13 +69,31 @@ export class CryptoHelper { }; }; - constructor(config: Config) { + constructor(private readonly config: Config) {} + + @OnEvent('config.init') + onConfigInit() { + this.setup(); + } + + @OnEvent('config.changed') + onConfigChanged(event: Events['config.changed']) { + if (event.updates.crypto?.privateKey) { + this.setup(); + } + } + + private setup() { + const key = this.config.crypto.privateKey || generatePrivateKey(); + const privateKey = readPrivateKey(key); + const publicKey = readPublicKey(key); + this.keyPair = { - publicKey: Buffer.from(config.crypto.secret.publicKey, 'utf8'), - privateKey: Buffer.from(config.crypto.secret.privateKey, 'utf8'), + publicKey: Buffer.from(publicKey), + privateKey: Buffer.from(privateKey), sha256: { - publicKey: this.sha256(config.crypto.secret.publicKey), - privateKey: this.sha256(config.crypto.secret.privateKey), + publicKey: this.sha256(publicKey), + privateKey: this.sha256(privateKey), }, }; } diff --git a/packages/backend/server/src/base/helpers/url.ts b/packages/backend/server/src/base/helpers/url.ts index 9a2bbf05a5..ed04aff89f 100644 --- a/packages/backend/server/src/base/helpers/url.ts +++ b/packages/backend/server/src/base/helpers/url.ts @@ -4,16 +4,23 @@ import { Injectable } from '@nestjs/common'; import type { Response } from 'express'; import { Config } from '../config'; +import { OnEvent } from '../event'; @Injectable() export class URLHelper { - private readonly redirectAllowHosts: string[]; + redirectAllowHosts!: string[]; - readonly origin: string; - readonly baseUrl: string; - readonly home: string; + origin!: string; + baseUrl!: string; + home!: string; constructor(private readonly config: Config) { + this.init(); + } + + @OnEvent('config.changed') + @OnEvent('config.init') + init() { if (this.config.server.externalUrl) { if (!this.verify(this.config.server.externalUrl)) { throw new Error( diff --git a/packages/backend/server/src/base/index.ts b/packages/backend/server/src/base/index.ts index 8e6ba4421a..9a92b4b275 100644 --- a/packages/backend/server/src/base/index.ts +++ b/packages/backend/server/src/base/index.ts @@ -6,17 +6,14 @@ export { SessionCache, } from './cache'; export { - type AFFiNEConfig, - applyEnvToConfig, Config, - type ConfigPaths, - DeploymentType, - getAFFiNEConfigModifier, + ConfigFactory, + defineModuleConfig, + type JSONSchema, } from './config'; export * from './error'; export { EventBus, OnEvent } from './event'; export { - type GraphqlContext, paginate, Paginated, PaginationInput, @@ -30,8 +27,12 @@ export { CallMetric, metrics } from './metrics'; export { Lock, Locker, Mutex, RequestMutex } from './mutex'; export * from './nestjs'; export { type PrismaTransaction } from './prisma'; -export { Runtime } from './runtime'; export * from './storage'; -export { type StorageProvider, StorageProviderFactory } from './storage'; +export { + autoMetadata, + type StorageProvider, + type StorageProviderConfig, + StorageProviderFactory, +} from './storage'; export { CloudThrottlerGuard, SkipThrottle, Throttle } from './throttler'; export * from './utils'; diff --git a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts index d0e309cbb3..e0c307a59d 100644 --- a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts +++ b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts @@ -52,13 +52,15 @@ class JobHandlers { test.before(async () => { module = await createTestingModule({ imports: [ - ConfigModule.forRoot({ + ConfigModule.override({ job: { worker: { - // NOTE(@forehalo): - // bullmq will hold the connection to check stalled jobs, - // which will keep the test process alive to timeout. - stalledInterval: 100, + defaultWorkerOptions: { + // NOTE(@forehalo): + // bullmq will hold the connection to check stalled jobs, + // which will keep the test process alive to timeout. + stalledInterval: 100, + }, }, queue: { defaultJobOptions: { delay: 1000 }, @@ -82,7 +84,7 @@ test.afterEach(async () => { // @ts-expect-error private api const inner = queue.getQueue('nightly'); await inner.obliterate({ force: true }); - inner.resume(); + await inner.resume(); }); test.after.always(async () => { @@ -132,7 +134,7 @@ test('should remove job from queue', async t => { // #region executor test('should start workers', async t => { // @ts-expect-error private api - const worker = executor.workers['nightly']; + const worker = executor.workers.get('nightly')!; t.truthy(worker); t.true(worker.isRunning()); diff --git a/packages/backend/server/src/base/job/queue/config.ts b/packages/backend/server/src/base/job/queue/config.ts index c912728dd5..51da1bb037 100644 --- a/packages/backend/server/src/base/job/queue/config.ts +++ b/packages/backend/server/src/base/job/queue/config.ts @@ -1,61 +1,86 @@ import { QueueOptions, WorkerOptions } from 'bullmq'; -import { - defineRuntimeConfig, - defineStartupConfig, - ModuleConfig, -} from '../../config'; +import { defineModuleConfig, JSONSchema } from '../../config'; import { Queue } from './def'; -declare module '../../config' { - interface AppConfig { - job: ModuleConfig< - { - queue: Omit; - worker: Omit; - }, - { - queues: { - [key in Queue]: { - concurrency: number; - }; - }; - } - >; +declare global { + interface AppConfigSchema { + job: { + queue: ConfigItem>; + worker: ConfigItem<{ + defaultWorkerOptions: Omit; + }>; + queues: { + [key in Queue]: ConfigItem<{ + concurrency: number; + }>; + }; + }; } } -defineStartupConfig('job', { +const schema: JSONSchema = { + type: 'object', + properties: { + concurrency: { type: 'number' }, + }, +}; + +defineModuleConfig('job', { queue: { - prefix: AFFiNE.node.test ? 'affine_job_test' : 'affine_job', - defaultJobOptions: { - attempts: 5, - // should remove job after it's completed, because we will add a new job with the same job id - removeOnComplete: true, - removeOnFail: { - age: 24 * 3600 /* 1 day */, - count: 500, + desc: 'The config for job queues', + default: { + prefix: env.testing ? 'affine_job_test' : 'affine_job', + defaultJobOptions: { + attempts: 5, + // should remove job after it's completed, because we will add a new job with the same job id + removeOnComplete: true, + removeOnFail: { + age: 24 * 3600 /* 1 day */, + count: 500, + }, }, }, + link: 'https://api.docs.bullmq.io/interfaces/v5.QueueOptions.html', }, - worker: {}, -}); -defineRuntimeConfig('job', { - 'queues.nightly.concurrency': { - default: 1, - desc: 'Concurrency of worker consuming of nightly checking job queue', + worker: { + desc: 'The config for job workers', + default: { + defaultWorkerOptions: {}, + }, + link: 'https://api.docs.bullmq.io/interfaces/v5.WorkerOptions.html', }, - 'queues.notification.concurrency': { - default: 10, - desc: 'Concurrency of worker consuming of notification job queue', + + 'queues.copilot': { + desc: 'The config for copilot job queue', + default: { + concurrency: 1, + }, + schema, }, - 'queues.doc.concurrency': { - default: 1, - desc: 'Concurrency of worker consuming of doc job queue', + + 'queues.doc': { + desc: 'The config for doc job queue', + default: { + concurrency: 1, + }, + schema, }, - 'queues.copilot.concurrency': { - default: 1, - desc: 'Concurrency of worker consuming of copilot job queue', + + 'queues.notification': { + desc: 'The config for notification job queue', + default: { + concurrency: 10, + }, + schema, + }, + + 'queues.nightly': { + desc: 'The config for nightly job queue', + default: { + concurrency: 1, + }, + schema, }, }); diff --git a/packages/backend/server/src/base/job/queue/def.ts b/packages/backend/server/src/base/job/queue/def.ts index 6db33e2908..c5994fb7f9 100644 --- a/packages/backend/server/src/base/job/queue/def.ts +++ b/packages/backend/server/src/base/job/queue/def.ts @@ -49,7 +49,7 @@ export const OnJob = (job: JobName) => { if (!QUEUES.includes(ns as Queue)) { throw new Error( `Invalid job queue: ${ns}, must be one of [${QUEUES.join(', ')}]. -If you want to introduce new job queue, please modify the Queue enum first in ${join(AFFiNE.projectRoot, 'src/base/job/queue/def.ts')}` +If you want to introduce new job queue, please modify the Queue enum first in ${join(env.projectRoot, 'src/base/job/queue/def.ts')}` ); } diff --git a/packages/backend/server/src/base/job/queue/executor.ts b/packages/backend/server/src/base/job/queue/executor.ts index a9e019b0c3..08f4003a27 100644 --- a/packages/backend/server/src/base/job/queue/executor.ts +++ b/packages/backend/server/src/base/job/queue/executor.ts @@ -1,49 +1,51 @@ -import { - Injectable, - Logger, - OnApplicationBootstrap, - OnApplicationShutdown, -} from '@nestjs/common'; +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { Worker } from 'bullmq'; -import { difference } from 'lodash-es'; +import { difference, merge } from 'lodash-es'; import { CLS_ID, ClsServiceManager } from 'nestjs-cls'; import { Config } from '../../config'; +import { OnEvent } from '../../event'; import { metrics, wrapCallMetric } from '../../metrics'; import { QueueRedis } from '../../redis'; -import { Runtime } from '../../runtime'; import { genRequestId } from '../../utils'; import { JOB_SIGNAL, namespace, Queue, QUEUES } from './def'; import { JobHandlerScanner } from './scanner'; @Injectable() -export class JobExecutor - implements OnApplicationBootstrap, OnApplicationShutdown -{ +export class JobExecutor implements OnModuleDestroy { private readonly logger = new Logger('job'); - private readonly workers: Record = {}; + private readonly workers: Map = new Map(); constructor( private readonly config: Config, private readonly redis: QueueRedis, - private readonly scanner: JobHandlerScanner, - private readonly runtime: Runtime + private readonly scanner: JobHandlerScanner ) {} - async onApplicationBootstrap() { - const queues = this.config.flavor.graphql - ? difference(QUEUES, [Queue.DOC]) - : []; + @OnEvent('config.init') + async onConfigInit() { + const queues = env.flavors.graphql ? difference(QUEUES, [Queue.DOC]) : []; // NOTE(@forehalo): only enable doc queue in doc service - if (this.config.flavor.doc) { + if (env.flavors.doc) { queues.push(Queue.DOC); } await this.startWorkers(queues); } - async onApplicationShutdown() { + @OnEvent('config.changed') + async onConfigChanged({ updates }: Events['config.changed']) { + if (updates.job?.queues) { + Object.entries(updates.job.queues).forEach(([queue, options]) => { + if (options.concurrency) { + this.setConcurrency(queue as Queue, options.concurrency); + } + }); + } + } + + async onModuleDestroy() { await this.stopWorkers(); } @@ -98,38 +100,35 @@ export class JobExecutor } } - private async startWorkers(queues: Queue[]) { - const configs = - (await this.runtime.fetchAll( - queues.reduce( - (ret, queue) => { - ret[`job/queues.${queue}.concurrency`] = true; - return ret; - }, - {} as { - [key in `job/queues.${Queue}.concurrency`]: true; - } - ) - // TODO(@forehalo): fix the override by [payment/service.spec.ts] - )) ?? {}; + setConcurrency(queue: Queue, concurrency: number) { + const worker = this.workers.get(queue); + if (!worker) { + throw new Error(`Worker for [${queue}] not found.`); + } + worker.concurrency = concurrency; + } + + private async startWorkers(queues: Queue[]) { for (const queue of queues) { - const concurrency = - (configs[`job/queues.${queue}.concurrency`] as number) ?? - this.config.job.worker.concurrency ?? - 1; + const queueOptions = this.config.job.queues[queue]; + const concurrency = queueOptions.concurrency ?? 1; const worker = new Worker( queue, async job => { await this.run(job.name as JobName, job.data); }, - { - ...this.config.job.queue, - ...this.config.job.worker, - connection: this.redis, - concurrency, - } + merge( + {}, + this.config.job.queue, + this.config.job.worker.defaultWorkerOptions, + queueOptions, + { + concurrency, + connection: this.redis, + } + ) ); worker.on('error', error => { @@ -140,13 +139,13 @@ export class JobExecutor `Queue Worker [${queue}] started; concurrency=${concurrency};` ); - this.workers[queue] = worker; + this.workers.set(queue, worker); } } private async stopWorkers() { await Promise.all( - Object.values(this.workers).map(async worker => { + Array.from(this.workers.values()).map(async worker => { await worker.close(true); }) ); diff --git a/packages/backend/server/src/base/metrics/config.ts b/packages/backend/server/src/base/metrics/config.ts index 7494089ac3..6ab228ba67 100644 --- a/packages/backend/server/src/base/metrics/config.ts +++ b/packages/backend/server/src/base/metrics/config.ts @@ -1,33 +1,16 @@ -import { defineStartupConfig, ModuleConfig } from '../config'; +import { defineModuleConfig } from '../config'; -declare module '../config' { - interface AppConfig { - metrics: ModuleConfig<{ - /** - * Enable metric and tracing collection - */ +declare global { + interface AppConfigSchema { + metrics: { enabled: boolean; - /** - * Enable telemetry - */ - telemetry: { - enabled: boolean; - token: string; - }; - customerIo: { - token: string; - }; - }>; + }; } } -defineStartupConfig('metrics', { - enabled: false, - telemetry: { - enabled: false, - token: '', - }, - customerIo: { - token: '', +defineModuleConfig('metrics', { + enabled: { + desc: 'Enable metric and tracing collection', + default: false, }, }); diff --git a/packages/backend/server/src/base/metrics/index.ts b/packages/backend/server/src/base/metrics/index.ts index ee2d98f292..b774930be3 100644 --- a/packages/backend/server/src/base/metrics/index.ts +++ b/packages/backend/server/src/base/metrics/index.ts @@ -1,54 +1,14 @@ import './config'; -import { - Global, - Module, - OnModuleDestroy, - OnModuleInit, - Provider, -} from '@nestjs/common'; -import { ModuleRef } from '@nestjs/core'; -import { NodeSDK } from '@opentelemetry/sdk-node'; +import { Global, Module } from '@nestjs/common'; -import { Config } from '../config'; -import { - LocalOpentelemetryFactory, - OpentelemetryFactory, - registerCustomMetrics, -} from './opentelemetry'; - -const factorProvider: Provider = { - provide: OpentelemetryFactory, - useFactory: (config: Config) => { - return config.metrics.enabled ? new LocalOpentelemetryFactory() : null; - }, - inject: [Config], -}; +import { OpentelemetryFactory } from './opentelemetry'; @Global() @Module({ - providers: [factorProvider], - exports: [factorProvider], + providers: [OpentelemetryFactory], }) -export class MetricsModule implements OnModuleInit, OnModuleDestroy { - private sdk: NodeSDK | null = null; - constructor(private readonly ref: ModuleRef) {} - - onModuleInit() { - const factor = this.ref.get(OpentelemetryFactory, { strict: false }); - if (factor) { - this.sdk = factor.create(); - this.sdk.start(); - registerCustomMetrics(); - } - } - - async onModuleDestroy() { - if (this.sdk) { - await this.sdk.shutdown(); - } - } -} +export class MetricsModule {} export * from './metrics'; export * from './utils'; diff --git a/packages/backend/server/src/base/metrics/metrics.ts b/packages/backend/server/src/base/metrics/metrics.ts index 74432c07cd..29f81db142 100644 --- a/packages/backend/server/src/base/metrics/metrics.ts +++ b/packages/backend/server/src/base/metrics/metrics.ts @@ -2,11 +2,28 @@ import { Gauge, Histogram, Meter, + MeterProvider, MetricOptions, + metrics as otelMetrics, UpDownCounter, } from '@opentelemetry/api'; +import { HostMetrics } from '@opentelemetry/host-metrics'; -import { getMeter } from './opentelemetry'; +function getMeterProvider() { + return otelMetrics.getMeterProvider(); +} + +export function registerCustomMetrics() { + const hostMetricsMonitoring = new HostMetrics({ + name: 'instance-host-metrics', + meterProvider: getMeterProvider() as MeterProvider, + }); + hostMetricsMonitoring.start(); +} + +export function getMeter(name = 'business') { + return getMeterProvider().getMeter(name); +} type MetricType = 'counter' | 'gauge' | 'histogram'; type Metric = T extends 'counter' @@ -122,5 +139,3 @@ export const metrics = new Proxy>( }, } ); - -export function stopMetrics() {} diff --git a/packages/backend/server/src/base/metrics/opentelemetry.ts b/packages/backend/server/src/base/metrics/opentelemetry.ts index 62f7db17e5..d22bdb736d 100644 --- a/packages/backend/server/src/base/metrics/opentelemetry.ts +++ b/packages/backend/server/src/base/metrics/opentelemetry.ts @@ -1,5 +1,4 @@ -import { OnModuleDestroy } from '@nestjs/common'; -import { metrics } from '@opentelemetry/api'; +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { CompositePropagator, W3CBaggagePropagator, @@ -7,7 +6,6 @@ import { } from '@opentelemetry/core'; import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; import { ZipkinExporter } from '@opentelemetry/exporter-zipkin'; -import { HostMetrics } from '@opentelemetry/host-metrics'; import { Instrumentation } from '@opentelemetry/instrumentation'; import { GraphQLInstrumentation } from '@opentelemetry/instrumentation-graphql'; import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; @@ -15,7 +13,6 @@ import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'; import { SocketIoInstrumentation } from '@opentelemetry/instrumentation-socket.io'; import { Resource } from '@opentelemetry/resources'; -import type { MeterProvider } from '@opentelemetry/sdk-metrics'; import { MetricProducer, MetricReader } from '@opentelemetry/sdk-metrics'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { @@ -30,11 +27,14 @@ import { } from '@opentelemetry/semantic-conventions/incubating'; import prismaInstrument from '@prisma/instrumentation'; +import { Config } from '../config'; +import { OnEvent } from '../event/def'; +import { registerCustomMetrics } from './metrics'; import { PrismaMetricProducer } from './prisma'; const { PrismaInstrumentation } = prismaInstrument; -export abstract class OpentelemetryFactory { +export abstract class BaseOpentelemetryFactory { abstract getMetricReader(): MetricReader; abstract getSpanExporter(): SpanExporter; @@ -55,9 +55,9 @@ export abstract class OpentelemetryFactory { getResource() { return new Resource({ - [ATTR_K8S_NAMESPACE_NAME]: AFFiNE.AFFINE_ENV, - [ATTR_SERVICE_NAME]: AFFiNE.flavor.type, - [ATTR_SERVICE_VERSION]: AFFiNE.version, + [ATTR_K8S_NAMESPACE_NAME]: env.NAMESPACE, + [ATTR_SERVICE_NAME]: env.FLAVOR, + [ATTR_SERVICE_VERSION]: env.version, }); } @@ -81,39 +81,58 @@ export abstract class OpentelemetryFactory { } } -export class LocalOpentelemetryFactory - extends OpentelemetryFactory +@Injectable() +export class OpentelemetryFactory + extends BaseOpentelemetryFactory implements OnModuleDestroy { - private readonly metricsExporter = new PrometheusExporter({ - metricProducers: this.getMetricsProducers(), - }); + private readonly logger = new Logger(OpentelemetryFactory.name); + #sdk: NodeSDK | null = null; + + constructor(private readonly config: Config) { + super(); + } + + @OnEvent('config.init') + async init(event: Events['config.init']) { + if (event.config.metrics.enabled) { + await this.setup(); + registerCustomMetrics(); + } + } + + @OnEvent('config.changed') + async onConfigChanged(event: Events['config.changed']) { + if ('metrics' in event.updates) { + await this.setup(); + } + } async onModuleDestroy() { - await this.metricsExporter.shutdown(); + await this.#sdk?.shutdown(); } override getMetricReader(): MetricReader { - return this.metricsExporter; + return new PrometheusExporter({ + metricProducers: this.getMetricsProducers(), + }); } override getSpanExporter(): SpanExporter { return new ZipkinExporter(); } -} -function getMeterProvider() { - return metrics.getMeterProvider(); -} - -export function registerCustomMetrics() { - const hostMetricsMonitoring = new HostMetrics({ - name: 'instance-host-metrics', - meterProvider: getMeterProvider() as MeterProvider, - }); - hostMetricsMonitoring.start(); -} - -export function getMeter(name = 'business') { - return getMeterProvider().getMeter(name); + private async setup() { + if (this.config.metrics.enabled) { + if (!this.#sdk) { + this.#sdk = this.create(); + } + this.#sdk.start(); + this.logger.log('OpenTelemetry SDK started'); + } else { + await this.#sdk?.shutdown(); + this.#sdk = null; + this.logger.log('OpenTelemetry SDK stopped'); + } + } } diff --git a/packages/backend/server/src/base/metrics/prisma.ts b/packages/backend/server/src/base/metrics/prisma.ts index 6c024c98ff..c4cffee600 100644 --- a/packages/backend/server/src/base/metrics/prisma.ts +++ b/packages/backend/server/src/base/metrics/prisma.ts @@ -10,7 +10,7 @@ import { ScopeMetrics, } from '@opentelemetry/sdk-metrics'; -import { PrismaService } from '../prisma'; +import { PrismaFactory } from '../prisma/factory'; function transformPrismaKey(key: string) { // replace first '_' to '/' as a scope prefix @@ -30,11 +30,11 @@ export class PrismaMetricProducer implements MetricProducer { errors: [], }; - if (!PrismaService.INSTANCE) { + if (!PrismaFactory.INSTANCE) { return result; } - const prisma = PrismaService.INSTANCE; + const prisma = PrismaFactory.INSTANCE; const endTime = hrTime(); diff --git a/packages/backend/server/src/base/nestjs/config.ts b/packages/backend/server/src/base/nestjs/config.ts deleted file mode 100644 index 46d867e9cb..0000000000 --- a/packages/backend/server/src/base/nestjs/config.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { defineStartupConfig, ModuleConfig } from '../../base/config'; - -export interface ServerStartupConfigurations { - /** - * Base url of AFFiNE server, used for generating external urls. - * default to be `[AFFiNE.protocol]://[AFFiNE.host][:AFFiNE.port]/[AFFiNE.path]` if not specified - */ - externalUrl: string; - /** - * Whether the server is hosted on a ssl enabled domain - */ - https: boolean; - /** - * where the server get deployed(FQDN). - */ - host: string; - /** - * which port the server will listen on - */ - port: number; - /** - * subpath where the server get deployed if there is. - */ - path: string; -} - -declare module '../../base/config' { - interface AppConfig { - server: ModuleConfig; - } -} - -defineStartupConfig('server', { - externalUrl: '', - https: false, - host: 'localhost', - port: 3010, - path: '', -}); diff --git a/packages/backend/server/src/base/nestjs/index.ts b/packages/backend/server/src/base/nestjs/index.ts index df5cfdc411..cfa91ee8dc 100644 --- a/packages/backend/server/src/base/nestjs/index.ts +++ b/packages/backend/server/src/base/nestjs/index.ts @@ -1,5 +1,3 @@ -import './config'; export * from './decorator'; export * from './exception'; -export * from './optional-module'; export * from './scanner'; diff --git a/packages/backend/server/src/base/nestjs/optional-module.ts b/packages/backend/server/src/base/nestjs/optional-module.ts deleted file mode 100644 index d7be4f5f90..0000000000 --- a/packages/backend/server/src/base/nestjs/optional-module.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { - DynamicModule, - Module, - ModuleMetadata, - Provider, - Type, -} from '@nestjs/common'; -import { omit } from 'lodash-es'; - -import type { AFFiNEConfig, ConfigPaths } from '../config'; - -export interface OptionalModuleMetadata extends ModuleMetadata { - /** - * Only install module if given config paths are defined in AFFiNE config. - */ - requires?: ConfigPaths[]; - - /** - * Only install module if the predication returns true. - */ - if?: (config: AFFiNEConfig) => boolean; - - /** - * Defines which feature will be enabled if the module installed. - */ - contributesTo?: import('../../core/config').ServerFeature; // avoid circlar dependency - - /** - * Defines which providers provided by other modules will be overridden if the module installed. - */ - overrides?: Provider[]; -} - -const additionalOptions = [ - 'contributesTo', - 'requires', - 'if', - 'overrides', -] as const satisfies Array; - -type OptionalDynamicModule = DynamicModule & OptionalModuleMetadata; - -export function OptionalModule(metadata: OptionalModuleMetadata) { - return (target: Type) => { - additionalOptions.forEach(option => { - if (Object.hasOwn(metadata, option)) { - Reflect.defineMetadata(option, metadata[option], target); - } - }); - - if (metadata.overrides) { - metadata.providers = (metadata.providers ?? []).concat( - metadata.overrides - ); - // eslint-disable-next-line @typescript-eslint/no-floating-promises - metadata.exports = (metadata.exports ?? []).concat(metadata.overrides); - } - - const nestMetadata = omit(metadata, additionalOptions); - Module(nestMetadata)(target); - }; -} - -export function getOptionalModuleMetadata< - T extends keyof OptionalModuleMetadata, ->(target: Type | OptionalDynamicModule, key: T): OptionalModuleMetadata[T] { - if ('module' in target) { - return target[key]; - } else { - return Reflect.getMetadata(key, target); - } -} diff --git a/packages/backend/server/src/base/prisma/config.ts b/packages/backend/server/src/base/prisma/config.ts index a5a9ce3ed3..b7d1b472d0 100644 --- a/packages/backend/server/src/base/prisma/config.ts +++ b/packages/backend/server/src/base/prisma/config.ts @@ -1,17 +1,27 @@ import type { Prisma } from '@prisma/client'; +import { z } from 'zod'; -import { defineStartupConfig, ModuleConfig } from '../config'; +import { defineModuleConfig } from '../config'; -interface PrismaStartupConfiguration extends Prisma.PrismaClientOptions { - datasourceUrl: string; -} - -declare module '../config' { - interface AppConfig { - prisma: ModuleConfig; +declare global { + interface AppConfigSchema { + db: { + datasourceUrl: string; + prisma: ConfigItem; + }; } } -defineStartupConfig('prisma', { - datasourceUrl: '', +defineModuleConfig('db', { + datasourceUrl: { + desc: 'The datasource url for the prisma client.', + default: 'postgresql://localhost:5432/affine', + env: 'DATABASE_URL', + shape: z.string().url(), + }, + prisma: { + desc: 'The config for the prisma client.', + default: {}, + link: 'https://www.prisma.io/docs/reference/api-reference/prisma-client-reference', + }, }); diff --git a/packages/backend/server/src/base/prisma/factory.ts b/packages/backend/server/src/base/prisma/factory.ts new file mode 100644 index 0000000000..e2bc99528a --- /dev/null +++ b/packages/backend/server/src/base/prisma/factory.ts @@ -0,0 +1,25 @@ +import type { OnModuleDestroy } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; + +import { Config } from '../config'; + +@Injectable() +export class PrismaFactory implements OnModuleDestroy { + static INSTANCE: PrismaClient | null = null; + readonly #instance: PrismaClient; + + constructor(config: Config) { + this.#instance = new PrismaClient(config.db.prisma); + PrismaFactory.INSTANCE = this.#instance; + } + + get() { + return this.#instance; + } + + async onModuleDestroy() { + await PrismaFactory.INSTANCE?.$disconnect(); + PrismaFactory.INSTANCE = null; + } +} diff --git a/packages/backend/server/src/base/prisma/index.ts b/packages/backend/server/src/base/prisma/index.ts index 4ea3ac8998..bf42d8d165 100644 --- a/packages/backend/server/src/base/prisma/index.ts +++ b/packages/backend/server/src/base/prisma/index.ts @@ -3,29 +3,24 @@ import './config'; import { Global, Module, Provider } from '@nestjs/common'; import { PrismaClient } from '@prisma/client'; -import { Config } from '../config'; -import { PrismaService } from './service'; +import { PrismaFactory } from './factory'; // only `PrismaClient` can be injected const clientProvider: Provider = { provide: PrismaClient, - useFactory: (config: Config) => { - if (PrismaService.INSTANCE) { - return PrismaService.INSTANCE; - } - - return new PrismaService(config.prisma); + useFactory: (factory: PrismaFactory) => { + return factory.get(); }, - inject: [Config], + inject: [PrismaFactory], }; @Global() @Module({ - providers: [clientProvider], + providers: [PrismaFactory, clientProvider], exports: [clientProvider], }) export class PrismaModule {} -export { PrismaService } from './service'; +export { PrismaFactory }; export type PrismaTransaction = Parameters< Parameters[0] diff --git a/packages/backend/server/src/base/prisma/service.ts b/packages/backend/server/src/base/prisma/service.ts deleted file mode 100644 index 281b67b8a7..0000000000 --- a/packages/backend/server/src/base/prisma/service.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; -import { Injectable } from '@nestjs/common'; -import { Prisma, PrismaClient } from '@prisma/client'; - -@Injectable() -export class PrismaService - extends PrismaClient - implements OnModuleInit, OnApplicationShutdown -{ - static INSTANCE: PrismaService | null = null; - - constructor(opts: Prisma.PrismaClientOptions) { - super(opts); - PrismaService.INSTANCE = this; - } - - async onModuleInit() { - await this.$connect(); - } - - async onApplicationShutdown(): Promise { - if (!AFFiNE.node.test) { - await this.$disconnect(); - PrismaService.INSTANCE = null; - } - } -} diff --git a/packages/backend/server/src/base/redis/config.ts b/packages/backend/server/src/base/redis/config.ts index 84fb350046..f921604e6e 100644 --- a/packages/backend/server/src/base/redis/config.ts +++ b/packages/backend/server/src/base/redis/config.ts @@ -1,11 +1,54 @@ import { RedisOptions } from 'ioredis'; +import { z } from 'zod'; -import { defineStartupConfig, ModuleConfig } from '../../base/config'; +import { defineModuleConfig } from '../config'; -declare module '../config' { - interface AppConfig { - redis: ModuleConfig; +declare global { + interface AppConfigSchema { + redis: { + host: string; + port: number; + db: number; + username: string; + password: string; + ioredis: ConfigItem< + Omit + >; + }; } } -defineStartupConfig('redis', {}); +defineModuleConfig('redis', { + db: { + desc: 'The database index of redis server to be used(Must be less than 10).', + default: 0, + env: ['REDIS_DATABASE', 'integer'], + validate: val => val >= 0 && val < 10, + }, + host: { + desc: 'The host of the redis server.', + default: 'localhost', + env: ['REDIS_HOST', 'string'], + }, + port: { + desc: 'The port of the redis server.', + default: 6379, + env: ['REDIS_PORT', 'integer'], + shape: z.number().positive(), + }, + username: { + desc: 'The username of the redis server.', + default: '', + env: ['REDIS_USERNAME', 'string'], + }, + password: { + desc: 'The password of the redis server.', + default: '', + env: ['REDIS_PASSWORD', 'string'], + }, + ioredis: { + desc: 'The config for the ioredis client.', + default: {}, + link: 'https://github.com/luin/ioredis', + }, +}); diff --git a/packages/backend/server/src/base/redis/instances.ts b/packages/backend/server/src/base/redis/instances.ts index 4f270dc67d..aca83c9d44 100644 --- a/packages/backend/server/src/base/redis/instances.ts +++ b/packages/backend/server/src/base/redis/instances.ts @@ -6,13 +6,10 @@ import { } from '@nestjs/common'; import { Redis as IORedis, RedisOptions } from 'ioredis'; -import { Config } from '../../base/config'; +import { Config } from '../config'; class Redis extends IORedis implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(this.constructor.name); - constructor(opts: RedisOptions) { - super(opts); - } errorHandler = (err: Error) => { this.logger.error(err); @@ -46,21 +43,29 @@ class Redis extends IORedis implements OnModuleInit, OnModuleDestroy { @Injectable() export class CacheRedis extends Redis { constructor(config: Config) { - super(config.redis); + super({ ...config.redis, ...config.redis.ioredis }); } } @Injectable() export class SessionRedis extends Redis { constructor(config: Config) { - super({ ...config.redis, db: (config.redis.db ?? 0) + 2 }); + super({ + ...config.redis, + ...config.redis.ioredis, + db: (config.redis.db ?? 0) + 2, + }); } } @Injectable() export class SocketIoRedis extends Redis { constructor(config: Config) { - super({ ...config.redis, db: (config.redis.db ?? 0) + 3 }); + super({ + ...config.redis, + ...config.redis.ioredis, + db: (config.redis.db ?? 0) + 3, + }); } } @@ -69,6 +74,7 @@ export class QueueRedis extends Redis { constructor(config: Config) { super({ ...config.redis, + ...config.redis.ioredis, db: (config.redis.db ?? 0) + 4, // required explicitly set to `null` by bullmq maxRetriesPerRequest: null, diff --git a/packages/backend/server/src/base/runtime/event.ts b/packages/backend/server/src/base/runtime/event.ts deleted file mode 100644 index 6697747a00..0000000000 --- a/packages/backend/server/src/base/runtime/event.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { FlattenedAppRuntimeConfig } from '../config/types'; - -declare global { - interface Events { - 'runtime.changed__NOT_IMPLEMENTED__': Partial; - } -} diff --git a/packages/backend/server/src/base/runtime/index.ts b/packages/backend/server/src/base/runtime/index.ts deleted file mode 100644 index 3e57b9184f..0000000000 --- a/packages/backend/server/src/base/runtime/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Global, Module } from '@nestjs/common'; - -import { Runtime } from './service'; - -@Global() -@Module({ - providers: [Runtime], - exports: [Runtime], -}) -export class RuntimeModule {} -export { Runtime }; diff --git a/packages/backend/server/src/base/runtime/service.ts b/packages/backend/server/src/base/runtime/service.ts deleted file mode 100644 index 9f401f21c4..0000000000 --- a/packages/backend/server/src/base/runtime/service.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { - forwardRef, - Inject, - Injectable, - Logger, - OnModuleInit, -} from '@nestjs/common'; -import { PrismaClient } from '@prisma/client'; -import { difference, keyBy } from 'lodash-es'; - -import { Cache } from '../cache'; -import { defaultRuntimeConfig, runtimeConfigType } from '../config/register'; -import { - AppRuntimeConfigModules, - FlattenedAppRuntimeConfig, -} from '../config/types'; -import { InvalidRuntimeConfigType, RuntimeConfigNotFound } from '../error'; -import { defer } from '../utils/promise'; - -function validateConfigType( - key: K, - value: any -) { - const config = defaultRuntimeConfig[key]; - - if (!config) { - throw new RuntimeConfigNotFound({ key }); - } - - const want = config.type; - const get = runtimeConfigType(value); - if (get !== want) { - throw new InvalidRuntimeConfigType({ - key, - want, - get, - }); - } -} - -/** - * runtime.fetch(k) // v1 - * runtime.fetchAll(k1, k2, k3) // [v1, v2, v3] - * runtime.set(k, v) - * runtime.update(k, (v) => { - * v.xxx = 'yyy'; - * return v - * }) - */ -@Injectable() -export class Runtime implements OnModuleInit { - private readonly logger = new Logger('App:RuntimeConfig'); - - constructor( - private readonly db: PrismaClient, - // circular deps: runtime => cache => redis(maybe) => config => runtime - @Inject(forwardRef(() => Cache)) private readonly cache: Cache - ) {} - - async onModuleInit() { - await this.upgradeDB(); - } - - async fetch( - k: K - ): Promise { - const cached = await this.loadCache(k); - - if (cached !== undefined) { - return cached; - } - - const dbValue = await this.loadDb(k); - - if (dbValue === undefined) { - throw new RuntimeConfigNotFound({ key: k }); - } - - await this.setCache(k, dbValue); - - return dbValue; - } - - async fetchAll< - Selector extends { [Key in keyof FlattenedAppRuntimeConfig]?: true }, - >( - selector: Selector - ): Promise<{ - // @ts-expect-error allow - [Key in keyof Selector]: FlattenedAppRuntimeConfig[Key]; - }> { - const keys = Object.keys(selector); - - if (keys.length === 0) { - return {} as any; - } - - const records = await this.db.runtimeConfig.findMany({ - select: { - id: true, - value: true, - }, - where: { - id: { - in: keys, - }, - deletedAt: null, - }, - }); - - const keyed = keyBy(records, 'id'); - return keys.reduce((ret, key) => { - ret[key] = keyed[key]?.value ?? defaultRuntimeConfig[key].value; - return ret; - }, {} as any); - } - - async list(module?: AppRuntimeConfigModules) { - return await this.db.runtimeConfig.findMany({ - where: module ? { module, deletedAt: null } : { deletedAt: null }, - }); - } - - async set< - K extends keyof FlattenedAppRuntimeConfig, - V = FlattenedAppRuntimeConfig[K], - >(key: K, value: V) { - validateConfigType(key, value); - const config = await this.db.runtimeConfig.upsert({ - where: { - id: key, - deletedAt: null, - }, - create: { - ...defaultRuntimeConfig[key], - value: value as any, - }, - update: { - value: value as any, - deletedAt: null, - }, - }); - - await this.setCache(key, config.value as FlattenedAppRuntimeConfig[K]); - return config; - } - - async update< - K extends keyof FlattenedAppRuntimeConfig, - V = FlattenedAppRuntimeConfig[K], - >(k: K, modifier: (v: V) => V | Promise) { - const data = await this.fetch(k); - - const updated = await modifier(data as V); - - await this.set(k, updated); - - return updated; - } - - async loadDb( - k: K - ): Promise { - const v = await this.db.runtimeConfig.findFirst({ - where: { - id: k, - deletedAt: null, - }, - }); - - if (v) { - return v.value as FlattenedAppRuntimeConfig[K]; - } else { - const record = await this.db.runtimeConfig.create({ - data: defaultRuntimeConfig[k], - }); - - return record.value as any; - } - } - - async loadCache( - k: K - ): Promise { - return this.cache.get(`SERVER_RUNTIME:${k}`); - } - - async setCache( - k: K, - v: FlattenedAppRuntimeConfig[K] - ): Promise { - return this.cache.set( - `SERVER_RUNTIME:${k}`, - v, - { ttl: 60 * 1000 } - ); - } - - /** - * Upgrade the DB with latest runtime configs - */ - private async upgradeDB() { - const existingConfig = await this.db.runtimeConfig.findMany({ - select: { - id: true, - }, - where: { - deletedAt: null, - }, - }); - - const defined = Object.keys(defaultRuntimeConfig); - const existing = existingConfig.map(c => c.id); - const newConfigs = difference(defined, existing); - const deleteConfigs = difference(existing, defined); - - if (!newConfigs.length && !deleteConfigs.length) { - return; - } - - this.logger.log(`Found runtime config changes, upgrading...`); - const acquired = await this.cache.setnx('runtime:upgrade', 1, { - ttl: 10 * 60 * 1000, - }); - await using _ = defer(async () => { - await this.cache.delete('runtime:upgrade'); - }); - - if (acquired) { - for (const key of newConfigs) { - await this.db.runtimeConfig.upsert({ - create: defaultRuntimeConfig[key], - // old deleted setting should be restored - update: { - ...defaultRuntimeConfig[key], - deletedAt: null, - }, - where: { - id: key, - }, - }); - } - - await this.db.runtimeConfig.updateMany({ - where: { - id: { - in: deleteConfigs, - }, - }, - data: { - deletedAt: new Date(), - }, - }); - } - - this.logger.log('Upgrade completed'); - } -} diff --git a/packages/backend/server/src/base/storage/config.ts b/packages/backend/server/src/base/storage/config.ts deleted file mode 100644 index fde09ec803..0000000000 --- a/packages/backend/server/src/base/storage/config.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { homedir } from 'node:os'; -import { join } from 'node:path'; - -import { defineStartupConfig, ModuleConfig } from '../config'; - -export interface FsStorageConfig { - path: string; -} - -export interface StorageProvidersConfig { - fs?: FsStorageConfig; -} - -declare module '../config' { - interface AppConfig { - storageProviders: ModuleConfig; - } -} - -defineStartupConfig('storageProviders', { - fs: { - path: join(homedir(), '.affine/storage'), - }, -}); - -export type StorageProviderType = keyof StorageProvidersConfig; - -export type StorageConfig = { - provider: StorageProviderType; - bucket: string; -} & Ext; - -export interface StoragesConfig { - avatar: StorageConfig<{ publicLinkFactory: (key: string) => string }>; - blob: StorageConfig; - copilot: StorageConfig; -} - -export interface AFFiNEStorageConfig { - /** - * All providers for object storage - * - * Support different providers for different usage at the same time. - */ - providers: StorageProvidersConfig; - storages: StoragesConfig; -} - -export type StorageProviders = AFFiNEStorageConfig['providers']; -export type Storages = keyof AFFiNEStorageConfig['storages']; - -export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig { - return { - providers: { - fs: { - path: join(homedir(), '.affine/storage'), - }, - }, - storages: { - avatar: { - provider: 'fs', - bucket: 'avatars', - publicLinkFactory: key => `/api/avatars/${key}`, - }, - blob: { - provider: 'fs', - bucket: 'blobs', - }, - copilot: { - provider: 'fs', - bucket: 'copilot', - }, - }, - }; -} diff --git a/packages/backend/server/src/base/storage/factory.ts b/packages/backend/server/src/base/storage/factory.ts new file mode 100644 index 0000000000..16b63dcfde --- /dev/null +++ b/packages/backend/server/src/base/storage/factory.ts @@ -0,0 +1,20 @@ +import { Injectable } from '@nestjs/common'; + +import { + StorageProvider, + StorageProviderConfig, + StorageProviders, +} from './providers'; + +@Injectable() +export class StorageProviderFactory { + create(config: StorageProviderConfig): StorageProvider { + const Provider = StorageProviders[config.provider]; + + if (!Provider) { + throw new Error(`Unknown storage provider type: ${config.provider}`); + } + + return new Provider(config.config, config.bucket); + } +} diff --git a/packages/backend/server/src/base/storage/index.ts b/packages/backend/server/src/base/storage/index.ts index 7804f00869..d0abd0d92e 100644 --- a/packages/backend/server/src/base/storage/index.ts +++ b/packages/backend/server/src/base/storage/index.ts @@ -1,17 +1,6 @@ -import './config'; - import { Global, Module } from '@nestjs/common'; -import { registerStorageProvider, StorageProviderFactory } from './providers'; -import { FsStorageProvider } from './providers/fs'; - -registerStorageProvider('fs', (config, bucket) => { - if (!config.storageProviders.fs) { - throw new Error('Missing fs storage provider configuration'); - } - - return new FsStorageProvider(config.storageProviders.fs, bucket); -}); +import { StorageProviderFactory } from './factory'; @Global() @Module({ @@ -19,16 +8,5 @@ registerStorageProvider('fs', (config, bucket) => { exports: [StorageProviderFactory], }) export class StorageProviderModule {} - -export * from '../../native'; -export type { StorageProviderType } from './config'; -export type { - BlobInputType, - BlobOutputType, - GetObjectMetadata, - ListObjectsMetadata, - PutObjectMetadata, - StorageProvider, -} from './providers'; -export { registerStorageProvider, StorageProviderFactory } from './providers'; -export { autoMetadata, toBuffer } from './providers/utils'; +export { StorageProviderFactory } from './factory'; +export * from './providers'; diff --git a/packages/backend/server/src/base/storage/providers/fs.ts b/packages/backend/server/src/base/storage/providers/fs.ts index 081e678aca..fc6252f72c 100644 --- a/packages/backend/server/src/base/storage/providers/fs.ts +++ b/packages/backend/server/src/base/storage/providers/fs.ts @@ -10,12 +10,12 @@ import { statSync, writeFileSync, } from 'node:fs'; -import { join, parse, resolve } from 'node:path'; +import { homedir } from 'node:os'; +import { join, parse } from 'node:path'; import { Readable } from 'node:stream'; import { Logger } from '@nestjs/common'; -import { FsStorageConfig } from '../config'; import { BlobInputType, GetObjectMetadata, @@ -30,6 +30,10 @@ function escapeKey(key: string): string { return key.replace(/\.?\.[/\\]/g, '%'); } +export interface FsStorageConfig { + path: string; +} + export class FsStorageProvider implements StorageProvider { private readonly path: string; private readonly logger: Logger; @@ -40,7 +44,9 @@ export class FsStorageProvider implements StorageProvider { config: FsStorageConfig, public readonly bucket: string ) { - this.path = resolve(config.path, bucket); + this.path = config.path.startsWith('~/') + ? join(homedir(), config.path.slice(2), bucket) + : join(config.path, bucket); this.ensureAvailability(); this.logger = new Logger(`${FsStorageProvider.name}:${bucket}`); diff --git a/packages/backend/server/src/base/storage/providers/index.ts b/packages/backend/server/src/base/storage/providers/index.ts index 5ee118c7ed..01be8e9fd6 100644 --- a/packages/backend/server/src/base/storage/providers/index.ts +++ b/packages/backend/server/src/base/storage/providers/index.ts @@ -1,34 +1,116 @@ -import { Injectable } from '@nestjs/common'; +import { Type } from '@nestjs/common'; -import { Config } from '../../config'; -import { StorageConfig, StorageProviderType } from '../config'; -import type { StorageProvider } from './provider'; +import { JSONSchema } from '../../config'; +import { FsStorageConfig, FsStorageProvider } from './fs'; +import { StorageProvider } from './provider'; +import { R2StorageConfig, R2StorageProvider } from './r2'; +import { S3StorageConfig, S3StorageProvider } from './s3'; -const availableProviders = new Map< - StorageProviderType, - (config: Config, bucket: string) => StorageProvider ->(); +export type StorageProviderName = 'fs' | 'aws-s3' | 'cloudflare-r2'; +export const StorageProviders: Record< + StorageProviderName, + Type +> = { + fs: FsStorageProvider, + 'aws-s3': S3StorageProvider, + 'cloudflare-r2': R2StorageProvider, +}; -export function registerStorageProvider( - type: StorageProviderType, - providerFactory: (config: Config, bucket: string) => StorageProvider -) { - availableProviders.set(type, providerFactory); -} - -@Injectable() -export class StorageProviderFactory { - constructor(private readonly config: Config) {} - - create(storage: StorageConfig): StorageProvider { - const providerFactory = availableProviders.get(storage.provider); - - if (!providerFactory) { - throw new Error(`Unknown storage provider type: ${storage.provider}`); +export type StorageProviderConfig = { bucket: string } & ( + | { + provider: 'fs'; + config: FsStorageConfig; } + | { + provider: 'aws-s3'; + config: S3StorageConfig; + } + | { + provider: 'cloudflare-r2'; + config: R2StorageConfig; + } +); - return providerFactory(this.config, storage.bucket); - } -} +const S3ConfigSchema: JSONSchema = { + type: 'object', + description: + 'The config for the s3 compatible storage provider. directly passed to aws-sdk client.\n@link https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html', + properties: { + credentials: { + type: 'object', + description: 'The credentials for the s3 compatible storage provider.', + properties: { + accessKeyId: { + type: 'string', + }, + secretAccessKey: { + type: 'string', + }, + }, + }, + }, +}; + +export const StorageJSONSchema: JSONSchema = { + oneOf: [ + { + type: 'object', + properties: { + provider: { + type: 'string', + enum: ['fs'], + }, + bucket: { + type: 'string', + }, + config: { + type: 'object', + properties: { + path: { + type: 'string', + }, + }, + }, + }, + }, + { + type: 'object', + properties: { + provider: { + type: 'string', + enum: ['aws-s3'], + }, + bucket: { + type: 'string', + }, + config: S3ConfigSchema, + }, + }, + { + type: 'object', + properties: { + provider: { + type: 'string', + enum: ['cloudflare-r2'], + }, + bucket: { + type: 'string', + }, + config: { + ...S3ConfigSchema, + properties: { + ...S3ConfigSchema.properties, + accountId: { + type: 'string' as const, + description: + 'The account id for the cloudflare r2 storage provider.', + }, + }, + }, + }, + }, + ], +}; export type * from './provider'; +export { autoMetadata, toBuffer } from './utils'; diff --git a/packages/backend/server/src/base/storage/providers/provider.ts b/packages/backend/server/src/base/storage/providers/provider.ts index 06d8009773..06b8ac4bfb 100644 --- a/packages/backend/server/src/base/storage/providers/provider.ts +++ b/packages/backend/server/src/base/storage/providers/provider.ts @@ -1,7 +1,5 @@ import type { Readable } from 'node:stream'; -import { StorageProviderType } from '../config'; - export interface GetObjectMetadata { /** * @default 'application/octet-stream' @@ -28,7 +26,6 @@ export type BlobInputType = Buffer | Readable | string; export type BlobOutputType = Readable; export interface StorageProvider { - readonly type: StorageProviderType; put( key: string, body: BlobInputType, diff --git a/packages/backend/server/src/plugins/storage/providers/r2.ts b/packages/backend/server/src/base/storage/providers/r2.ts similarity index 62% rename from packages/backend/server/src/plugins/storage/providers/r2.ts rename to packages/backend/server/src/base/storage/providers/r2.ts index efdae3400c..e7fe8f100d 100644 --- a/packages/backend/server/src/plugins/storage/providers/r2.ts +++ b/packages/backend/server/src/base/storage/providers/r2.ts @@ -2,12 +2,13 @@ import assert from 'node:assert'; import { Logger } from '@nestjs/common'; -import type { R2StorageConfig } from '../config'; -import { S3StorageProvider } from './s3'; +import { S3StorageConfig, S3StorageProvider } from './s3'; + +export interface R2StorageConfig extends S3StorageConfig { + accountId: string; +} export class R2StorageProvider extends S3StorageProvider { - override readonly type = 'cloudflare-r2' as any /* cast 'r2' to 's3' */; - constructor(config: R2StorageConfig, bucket: string) { assert(config.accountId, 'accountId is required for R2 storage provider'); super( @@ -15,6 +16,9 @@ export class R2StorageProvider extends S3StorageProvider { ...config, forcePathStyle: true, endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`, + // see https://github.com/aws/aws-sdk-js-v3/issues/6810 + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', }, bucket ); diff --git a/packages/backend/server/src/plugins/storage/providers/s3.ts b/packages/backend/server/src/base/storage/providers/s3.ts similarity index 97% rename from packages/backend/server/src/plugins/storage/providers/s3.ts rename to packages/backend/server/src/base/storage/providers/s3.ts index d560b2fa7d..7db115dcc0 100644 --- a/packages/backend/server/src/plugins/storage/providers/s3.ts +++ b/packages/backend/server/src/base/storage/providers/s3.ts @@ -9,26 +9,25 @@ import { NoSuchKey, PutObjectCommand, S3Client, + S3ClientConfig, } from '@aws-sdk/client-s3'; import { Logger } from '@nestjs/common'; import { - autoMetadata, BlobInputType, GetObjectMetadata, ListObjectsMetadata, PutObjectMetadata, StorageProvider, - toBuffer, -} from '../../../base/storage'; -import type { S3StorageConfig } from '../config'; +} from './provider'; +import { autoMetadata, toBuffer } from './utils'; + +export type S3StorageConfig = S3ClientConfig; export class S3StorageProvider implements StorageProvider { protected logger: Logger; protected client: S3Client; - readonly type = 'aws-s3'; - constructor( config: S3StorageConfig, public readonly bucket: string diff --git a/packages/backend/server/src/base/throttler/config.ts b/packages/backend/server/src/base/throttler/config.ts index 61c32ef4d7..1ae00ed2b0 100644 --- a/packages/backend/server/src/base/throttler/config.ts +++ b/packages/backend/server/src/base/throttler/config.ts @@ -1,27 +1,38 @@ -import { defineStartupConfig, ModuleConfig } from '../config'; +import { defineModuleConfig } from '../config'; export type ThrottlerType = 'default' | 'strict'; -type ThrottlerStartupConfigurations = { - [key in ThrottlerType]: { - ttl: number; - limit: number; - }; -}; - -declare module '../config' { - interface AppConfig { - throttler: ModuleConfig; +declare global { + interface AppConfigSchema { + throttle: { + enabled: boolean; + throttlers: { + [key in ThrottlerType]: ConfigItem<{ + ttl: number; + limit: number; + }>; + }; + }; } } -defineStartupConfig('throttler', { - default: { - ttl: 60, - limit: 120, +defineModuleConfig('throttle', { + enabled: { + desc: 'Whether the throttler is enabled.', + default: true, }, - strict: { - ttl: 60, - limit: 20, + 'throttlers.default': { + desc: 'The config for the default throttler.', + default: { + ttl: 60, + limit: 120, + }, + }, + 'throttlers.strict': { + desc: 'The config for the strict throttler.', + default: { + ttl: 60, + limit: 20, + }, }, }); diff --git a/packages/backend/server/src/base/throttler/index.ts b/packages/backend/server/src/base/throttler/index.ts index 04fb1ab2a9..a20174aa8e 100644 --- a/packages/backend/server/src/base/throttler/index.ts +++ b/packages/backend/server/src/base/throttler/index.ts @@ -24,14 +24,19 @@ export class ThrottlerStorage extends ThrottlerStorageService {} @Injectable() class CustomOptionsFactory implements ThrottlerOptionsFactory { - constructor(private readonly storage: ThrottlerStorage) {} + constructor( + private readonly config: Config, + private readonly storage: ThrottlerStorage + ) {} createThrottlerOptions() { const options: ThrottlerModuleOptions = { - throttlers: Object.entries(AFFiNE.throttler).map(([name, config]) => ({ - name, - ...config, - })), + throttlers: Object.entries(this.config.throttle.throttlers).map( + ([name, config]) => ({ + name, + ...config, + }) + ), storage: this.storage, }; @@ -84,6 +89,7 @@ export class CloudThrottlerGuard extends ThrottlerGuard { ttl, blockDuration, } = request; + let limit = request.limit; // give it 'default' if no throttler is specified, @@ -110,13 +116,9 @@ export class CloudThrottlerGuard extends ThrottlerGuard { let tracker = await this.getTracker(req); - if (this.config.node.dev) { - limit = Number.MAX_SAFE_INTEGER; - } else { - // custom limit or ttl APIs will be treated standalone - if (limit !== throttlerOptions.limit || ttl !== throttlerOptions.ttl) { - tracker += ';custom'; - } + // custom limit or ttl APIs will be treated standalone + if (limit !== throttlerOptions.limit || ttl !== throttlerOptions.ttl) { + tracker += ';custom'; } const key = this.generateKey( @@ -151,6 +153,10 @@ export class CloudThrottlerGuard extends ThrottlerGuard { } override async canActivate(context: ExecutionContext): Promise { + if (!this.config.throttle.enabled) { + return true; + } + const { req } = this.getRequestResponse(context); const throttler = this.getSpecifiedThrottler(context); diff --git a/packages/backend/server/src/base/utils/request.ts b/packages/backend/server/src/base/utils/request.ts index b957f26899..af71b78300 100644 --- a/packages/backend/server/src/base/utils/request.ts +++ b/packages/backend/server/src/base/utils/request.ts @@ -94,7 +94,7 @@ export function parseCookies( export type RequestType = GqlContextType | 'event' | 'job'; export function genRequestId(type: RequestType) { - return `${AFFiNE.flavor.type}:${type}:${randomUUID()}`; + return `${env.DEPLOYMENT_TYPE}:${type}:${randomUUID()}`; } export function getOrGenRequestId(type: RequestType) { diff --git a/packages/backend/server/src/base/utils/types.ts b/packages/backend/server/src/base/utils/types.ts index ddc422a3ad..f786ff4f2f 100644 --- a/packages/backend/server/src/base/utils/types.ts +++ b/packages/backend/server/src/base/utils/types.ts @@ -2,9 +2,7 @@ import { Readable } from 'node:stream'; export function ApplyType(): ConstructorOf { // @ts-expect-error used to fake the type of config - return class Inner implements T { - constructor() {} - }; + return class Inner implements T {}; } export type PathType = @@ -30,7 +28,7 @@ export type Join = Prefix extends string | number export type LeafPaths< T, - Path extends string = '', + Prefix extends string = '', MaxDepth extends string = '.....', Depth extends string = '', > = Depth extends MaxDepth @@ -40,7 +38,9 @@ export type LeafPaths< [K in keyof T]-?: K extends string | number ? T[K] extends PrimitiveType ? K - : Join> + : T[K] extends { __leaf: true } + ? K + : Join> : never; }[keyof T] : never; diff --git a/packages/backend/server/src/base/websocket/adapter.ts b/packages/backend/server/src/base/websocket/adapter.ts index b466d557fc..7387c1d8d1 100644 --- a/packages/backend/server/src/base/websocket/adapter.ts +++ b/packages/backend/server/src/base/websocket/adapter.ts @@ -1,7 +1,7 @@ import { INestApplication } from '@nestjs/common'; import { IoAdapter } from '@nestjs/platform-socket.io'; import { createAdapter } from '@socket.io/redis-adapter'; -import { Server } from 'socket.io'; +import { Server, Socket } from 'socket.io'; import { Config } from '../config'; import { AuthenticationRequired } from '../error'; @@ -14,7 +14,9 @@ export class SocketIoAdapter extends IoAdapter { } override createIOServer(port: number, options?: any): Server { - const config = this.app.get(WEBSOCKET_OPTIONS) as Config['websocket']; + const config = this.app.get(WEBSOCKET_OPTIONS) as Config['websocket'] & { + canActivate: (socket: Socket) => Promise; + }; const server: Server = super.createIOServer(port, { ...config, ...options, @@ -22,7 +24,6 @@ export class SocketIoAdapter extends IoAdapter { if (config.canActivate) { server.use((socket, next) => { - // @ts-expect-error checked config .canActivate(socket) .then(pass => { diff --git a/packages/backend/server/src/base/websocket/config.ts b/packages/backend/server/src/base/websocket/config.ts index 9732f4c435..f64f37e9a3 100644 --- a/packages/backend/server/src/base/websocket/config.ts +++ b/packages/backend/server/src/base/websocket/config.ts @@ -1,20 +1,34 @@ import { GatewayMetadata } from '@nestjs/websockets'; -import { Socket } from 'socket.io'; +import { z } from 'zod'; -import { defineStartupConfig, ModuleConfig } from '../config'; +import { defineModuleConfig } from '../config'; -declare module '../config' { - interface AppConfig { - websocket: ModuleConfig< - GatewayMetadata & { - canActivate?: (socket: Socket) => Promise; - } - >; +declare global { + interface AppConfigSchema { + websocket: { + transports: ConfigItem; + maxHttpBufferSize: number; + }; } } -defineStartupConfig('websocket', { - transports: ['websocket', 'polling'], - // see: https://socket.io/docs/v4/server-options/#maxhttpbuffersize - maxHttpBufferSize: 1e8, // 100 MB +defineModuleConfig('websocket', { + transports: { + desc: 'The enabled transports for accepting websocket traffics.', + default: ['websocket', 'polling'], + shape: z.array(z.enum(['websocket', 'polling'])), + schema: { + type: 'array', + items: { + type: 'string', + enum: ['websocket', 'polling'], + }, + }, + link: 'https://docs.nestjs.com/websockets/gateways#transports', + }, + maxHttpBufferSize: { + desc: 'How many bytes or characters a message can be, before closing the session (to avoid DoS).', + default: 1e8, // 100 MB + shape: z.number().int().positive(), + }, }); diff --git a/packages/backend/server/src/config/affine.env.ts b/packages/backend/server/src/config/affine.env.ts deleted file mode 100644 index a1931cb74c..0000000000 --- a/packages/backend/server/src/config/affine.env.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Convenient way to map environment variables to config values. -AFFiNE.ENV_MAP = { - AFFINE_SERVER_EXTERNAL_URL: ['server.externalUrl'], - AFFINE_SERVER_PORT: ['server.port', 'int'], - AFFINE_SERVER_HOST: 'server.host', - AFFINE_SERVER_SUB_PATH: 'server.path', - AFFINE_SERVER_HTTPS: ['server.https', 'boolean'], - ENABLE_TELEMETRY: ['metrics.telemetry.enabled', 'boolean'], - MAILER_HOST: 'mailer.host', - MAILER_PORT: ['mailer.port', 'int'], - MAILER_USER: 'mailer.auth.user', - MAILER_PASSWORD: 'mailer.auth.pass', - MAILER_SENDER: 'mailer.from.address', - MAILER_SECURE: ['mailer.secure', 'boolean'], - DATABASE_URL: 'prisma.datasourceUrl', - OAUTH_GOOGLE_CLIENT_ID: 'plugins.oauth.providers.google.clientId', - OAUTH_GOOGLE_CLIENT_SECRET: 'plugins.oauth.providers.google.clientSecret', - OAUTH_GITHUB_CLIENT_ID: 'plugins.oauth.providers.github.clientId', - OAUTH_GITHUB_CLIENT_SECRET: 'plugins.oauth.providers.github.clientSecret', - OAUTH_OIDC_ISSUER: 'plugins.oauth.providers.oidc.issuer', - OAUTH_OIDC_CLIENT_ID: 'plugins.oauth.providers.oidc.clientId', - OAUTH_OIDC_CLIENT_SECRET: 'plugins.oauth.providers.oidc.clientSecret', - OAUTH_OIDC_SCOPE: 'plugins.oauth.providers.oidc.args.scope', - OAUTH_OIDC_CLAIM_MAP_USERNAME: 'plugins.oauth.providers.oidc.args.claim_id', - OAUTH_OIDC_CLAIM_MAP_EMAIL: 'plugins.oauth.providers.oidc.args.claim_email', - OAUTH_OIDC_CLAIM_MAP_NAME: 'plugins.oauth.providers.oidc.args.claim_name', - METRICS_CUSTOMER_IO_TOKEN: ['metrics.customerIo.token', 'string'], - CAPTCHA_TURNSTILE_SECRET: ['plugins.captcha.turnstile.secret', 'string'], - COPILOT_OPENAI_API_KEY: 'plugins.copilot.openai.apiKey', - COPILOT_FAL_API_KEY: 'plugins.copilot.fal.apiKey', - COPILOT_GOOGLE_API_KEY: 'plugins.copilot.google.apiKey', - COPILOT_PERPLEXITY_API_KEY: 'plugins.copilot.perplexity.apiKey', - COPILOT_UNSPLASH_API_KEY: 'plugins.copilot.unsplashKey', - REDIS_SERVER_HOST: 'redis.host', - REDIS_SERVER_PORT: ['redis.port', 'int'], - REDIS_SERVER_USER: 'redis.username', - REDIS_SERVER_PASSWORD: 'redis.password', - REDIS_SERVER_DATABASE: ['redis.db', 'int'], - DOC_SERVICE_ENDPOINT: 'docService.endpoint', - STRIPE_API_KEY: 'plugins.payment.stripe.keys.APIKey', - STRIPE_WEBHOOK_KEY: 'plugins.payment.stripe.keys.webhookKey', -}; diff --git a/packages/backend/server/src/config/affine.self.ts b/packages/backend/server/src/config/affine.self.ts deleted file mode 100644 index af82f3957e..0000000000 --- a/packages/backend/server/src/config/affine.self.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* oxlint-disable @typescript-eslint/no-non-null-assertion */ -// Custom configurations for AFFiNE Cloud -// ==================================================================================== -// Q: WHY THIS FILE EXISTS? -// A: AFFiNE deployment environment may have a lot of custom environment variables, -// which are not suitable to be put in the `affine.ts` file. -// For example, AFFiNE Cloud Clusters are deployed on Google Cloud Platform. -// We need to enable the `gcloud` plugin to make sure the nodes working well, -// but the default selfhost version may not require it. -// So it's not a good idea to put such logic in the common `affine.ts` file. -// -// ``` -// if (AFFiNE.deploy) { -// AFFiNE.plugins.use('gcloud'); -// } -// ``` -// ==================================================================================== -const env = process.env; - -AFFiNE.serverName = AFFiNE.affine.canary - ? 'AFFiNE Canary Cloud' - : AFFiNE.affine.beta - ? 'AFFiNE Beta Cloud' - : 'AFFiNE Cloud'; -AFFiNE.metrics.enabled = !AFFiNE.node.test; - -if (env.R2_OBJECT_STORAGE_ACCOUNT_ID) { - AFFiNE.use('cloudflare-r2', { - accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID, - credentials: { - accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!, - secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!, - }, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - }); - AFFiNE.storages.avatar.provider = 'cloudflare-r2'; - AFFiNE.storages.avatar.bucket = 'account-avatar'; - AFFiNE.storages.avatar.publicLinkFactory = key => - `https://avatar.affineassets.com/${key}`; - - AFFiNE.storages.blob.provider = 'cloudflare-r2'; - AFFiNE.storages.blob.bucket = `workspace-blobs-${ - AFFiNE.affine.canary ? 'canary' : 'prod' - }`; - - AFFiNE.use('copilot', { - storage: { - provider: 'cloudflare-r2', - bucket: `workspace-copilot-${AFFiNE.affine.canary ? 'canary' : 'prod'}`, - }, - }); -} - -AFFiNE.use('copilot', { - openai: { - apiKey: '', - }, - fal: { - apiKey: '', - }, -}); -AFFiNE.use('payment', { - stripe: { - keys: { - // fake the key to ensure the server generate full GraphQL Schema even env vars are not set - APIKey: '1', - webhookKey: '1', - }, - }, -}); -AFFiNE.use('oauth'); - -/* Captcha Plugin Default Config */ -AFFiNE.use('captcha', { - turnstile: {}, - challenge: { - bits: 20, - }, -}); - -if (AFFiNE.deploy) { - AFFiNE.mailer = { - service: 'gmail', - auth: { - user: env.MAILER_USER, - pass: env.MAILER_PASSWORD, - }, - }; - - AFFiNE.use('gcloud'); -} else { - // only enable dev mode - AFFiNE.use('worker'); -} diff --git a/packages/backend/server/src/config/affine.ts b/packages/backend/server/src/config/affine.ts deleted file mode 100644 index fc34e43102..0000000000 --- a/packages/backend/server/src/config/affine.ts +++ /dev/null @@ -1,168 +0,0 @@ -// -// ############################################################### -// ## AFFiNE Configuration System ## -// ############################################################### -// Here is the file of all AFFiNE configurations that will affect runtime behavior. -// Override any configuration here and it will be merged when starting the server. -// Any changes in this file won't take effect before server restarted. -// -// -// ############################################################### -// ## General settings ## -// ############################################################### -// -// /* The name of AFFiNE Server, may show on the UI */ -AFFiNE.serverName = 'My Selfhosted AFFiNE Cloud'; -// -// /* Whether the server is deployed behind a HTTPS proxied environment */ -AFFiNE.server.https = false; -// /* Domain of your server that your server will be available at */ -AFFiNE.server.host = 'localhost'; -// /* The local port of your server that will listen on */ -AFFiNE.server.port = 3010; -// /* The sub path of your server */ -// /* For example, if you set `AFFiNE.server.path = '/affine'`, then the server will be available at `${domain}/affine` */ -// AFFiNE.server.path = '/affine'; -// /* The external URL of your server, will be consist of protocol + host + port by default */ -// /* Useful when you want to customize the link to server resources for example the doc share link or email link */ -// AFFiNE.server.externalUrl = 'http://affine.local:8080' -// -// -// ############################################################### -// ## Server Function settings ## -// ############################################################### -// -// /* Whether enable metrics and tracing while running the server */ -// /* The metrics will be available at `http://localhost:9464/metrics` with [Prometheus] format exported */ -// AFFiNE.metrics.enabled = true; -// -// -// AFFiNE.auth.session = { -// /* How long the login session would last by default */ -// ttl: 15 * 24 * 60 * 60, // 15 days -// /* How long we should refresh the token before it getting expired */ -// ttr: 7 * 24 * 60 * 60, // 7 days -// }; -// -// /* GraphQL configurations that control the behavior of the Apollo Server behind */ -// /* @see https://www.apollographql.com/docs/apollo-server/api/apollo-server */ -// AFFiNE.graphql = { -// /* Path to mount GraphQL API */ -// path: '/graphql', -// buildSchemaOptions: { -// numberScalarMode: 'integer', -// }, -// /* Whether allow client to query the schema introspection */ -// introspection: true, -// /* Whether enable GraphQL Playground UI */ -// playground: true, -// } -// -// /* Doc Store & Collaberation */ -// /* How long the buffer time of creating a new history snapshot when doc get updated */ -// AFFiNE.doc.history.interval = 1000 * 60 * 10; // 10 minutes -// -// /* How often the manager will start a new turn of merging pending updates into doc snapshot */ -// AFFiNE.doc.manager.updatePollInterval = 1000 * 3; -// -// -// ############################################################### -// ## Plugins settings ## -// ############################################################### -// -// /* Payment Plugin */ -// AFFiNE.use('payment', { -// stripe: { keys: {}, apiVersion: '2023-10-16' }, -// }); -// -// -// /* Captcha Plugin Default Config */ -// AFFiNE.plugins.use('captcha', { -// turnstile: {}, -// challenge: { -// bits: 20, -// }, -// }); -// -// -// /* Cloudflare R2 Plugin */ -// /* Enable if you choose to store workspace blobs or user avatars in Cloudflare R2 Storage Service */ -// AFFiNE.use('cloudflare-r2', { -// accountId: '', -// credentials: { -// accessKeyId: '', -// secretAccessKey: '', -// }, -// }); -// -// /* AWS S3 Plugin */ -// /* Enable if you choose to store workspace blobs or user avatars in AWS S3 Storage Service */ -// AFFiNE.use('aws-s3', { -// credentials: { -// accessKeyId: '', -// secretAccessKey: '', -// }, -// /* Whether enable checksum calculation for request */ -// /* see https://github.com/aws/aws-sdk-js-v3/issues/6810 */ -// requestChecksumCalculation: 'WHEN_REQUIRED', -// responseChecksumValidation: 'WHEN_REQUIRED', -// }) -// /* Update the provider of storages */ -// AFFiNE.storages.blob.provider = 'cloudflare-r2'; -// AFFiNE.storages.avatar.provider = 'cloudflare-r2'; -// -// /* OAuth Plugin */ -// AFFiNE.use('oauth', { -// providers: { -// github: { -// clientId: '', -// clientSecret: '', -// // See https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps -// args: { -// scope: 'user', -// }, -// }, -// google: { -// clientId: '', -// clientSecret: '', -// args: { -// // See https://developers.google.com/identity/protocols/oauth2 -// scope: 'openid email profile', -// promot: 'select_account', -// access_type: 'offline', -// }, -// }, -// oidc: { -// // OpenID Connect -// issuer: '', -// clientId: '', -// clientSecret: '', -// args: { -// scope: 'openid email profile', -// claim_id: 'preferred_username', -// claim_email: 'email', -// claim_name: 'name', -// }, -// }, -// }, -// }); -// -// /* Copilot Plugin */ -// AFFiNE.use('copilot', { -// openai: { -// apiKey: 'your-key', -// }, -// fal: { -// apiKey: 'your-key', -// }, -// unsplashKey: 'your-key', -// storage: { -// provider: 'cloudflare-r2', -// bucket: 'copilot', -// } -// }) -// -// /* AFFiNE Link Preview & Image Proxy API */ -// AFFiNE.use('worker', { -// allowedOrigin: ['example.com'], -// }); diff --git a/packages/backend/server/src/core/auth/config.ts b/packages/backend/server/src/core/auth/config.ts index fbe4c45a54..05e5695c6f 100644 --- a/packages/backend/server/src/core/auth/config.ts +++ b/packages/backend/server/src/core/auth/config.ts @@ -1,100 +1,69 @@ -import { - defineRuntimeConfig, - defineStartupConfig, - ModuleConfig, -} from '../../base/config'; +import { z } from 'zod'; -export interface AuthStartupConfigurations { - /** - * auth session config - */ +import { defineModuleConfig } from '../../base'; + +export interface AuthConfig { session: { - /** - * Application auth expiration time in seconds - */ ttl: number; - /** - * Application auth time to refresh in seconds - */ ttr: number; }; - - /** - * Application access token config - */ - accessToken: { - /** - * Application access token expiration time in seconds - */ - ttl: number; - /** - * Application refresh token expiration time in seconds - */ - refreshTokenTtl: number; - }; -} - -export interface AuthRuntimeConfigurations { - /** - * Whether allow anonymous users to sign up - */ allowSignup: boolean; - - /** - * Whether require email domain record verification before access restricted resources - */ requireEmailDomainVerification: boolean; - - /** - * Whether require email verification before access restricted resources - */ requireEmailVerification: boolean; - - /** - * The minimum and maximum length of the password when registering new users - */ - password: { + passwordRequirements: ConfigItem<{ min: number; max: number; - }; + }>; } -declare module '../../base/config' { - interface AppConfig { - auth: ModuleConfig; +declare global { + interface AppConfigSchema { + auth: AuthConfig; } } -defineStartupConfig('auth', { - session: { - ttl: 60 * 60 * 24 * 15, // 15 days - ttr: 60 * 60 * 24 * 7, // 7 days - }, - accessToken: { - ttl: 60 * 60 * 24 * 7, // 7 days - refreshTokenTtl: 60 * 60 * 24 * 30, // 30 days - }, -}); - -defineRuntimeConfig('auth', { +defineModuleConfig('auth', { allowSignup: { - desc: 'Whether allow new registrations', + desc: 'Whether allow new registrations.', default: true, }, requireEmailDomainVerification: { - desc: 'Whether require email domain record verification before accessing restricted resources', + desc: 'Whether require email domain record verification before accessing restricted resources.', default: false, }, requireEmailVerification: { - desc: 'Whether require email verification before accessing restricted resources', + desc: 'Whether require email verification before accessing restricted resources(not implemented).', default: true, }, - 'password.min': { - desc: 'The minimum length of user password', - default: 8, + passwordRequirements: { + desc: 'The password strength requirements when set new password.', + default: { + min: 8, + max: 32, + }, + shape: z + .object({ + min: z.number().min(1), + max: z.number().max(100), + }) + .strict() + .refine(data => data.min < data.max, { + message: 'Minimum length of password must be less than maximum length', + }), + schema: { + type: 'object', + properties: { + min: { type: 'number' }, + max: { type: 'number' }, + }, + }, }, - 'password.max': { - desc: 'The maximum length of user password', - default: 32, + 'session.ttl': { + desc: 'Application auth expiration time in seconds.', + default: 60 * 60 * 24 * 15, // 15 days + }, + 'session.ttr': { + desc: 'Application auth time to refresh in seconds.', + default: 60 * 60 * 24 * 7, // 7 days }, }); diff --git a/packages/backend/server/src/core/auth/controller.ts b/packages/backend/server/src/core/auth/controller.ts index 02b7280333..659c7ccbf0 100644 --- a/packages/backend/server/src/core/auth/controller.ts +++ b/packages/backend/server/src/core/auth/controller.ts @@ -23,7 +23,6 @@ import { InvalidAuthState, InvalidEmail, InvalidEmailToken, - Runtime, SignUpForbidden, Throttle, URLHelper, @@ -66,11 +65,10 @@ export class AuthController { private readonly auth: AuthService, private readonly models: Models, private readonly config: Config, - private readonly runtime: Runtime, private readonly cache: Cache, private readonly crypto: CryptoHelper ) { - if (config.node.dev) { + if (env.dev) { // set DNS servers in dev mode // NOTE: some network debugging software uses DNS hijacking // to better debug traffic, but their DNS servers may not @@ -93,7 +91,7 @@ export class AuthController { const user = await this.models.user.getUserByEmail(params.email); - const magicLinkAvailable = !!this.config.mailer.host; + const magicLinkAvailable = this.config.mailer.enabled; if (!user) { return { @@ -171,15 +169,11 @@ export class AuthController { // send email magic link const user = await this.models.user.getUserByEmail(email); if (!user) { - const allowSignup = await this.runtime.fetch('auth/allowSignup'); - if (!allowSignup) { + if (!this.config.auth.allowSignup) { throw new SignUpForbidden(); } - const requireEmailDomainVerification = await this.runtime.fetch( - 'auth/requireEmailDomainVerification' - ); - if (requireEmailDomainVerification) { + if (this.config.auth.requireEmailDomainVerification) { // verify domain has MX, SPF, DMARC records const [name, domain, ...rest] = email.split('@'); if (rest.length || !domain) { @@ -229,7 +223,7 @@ export class AuthController { } : {}), }); - if (this.config.node.dev) { + if (env.dev) { // make it easier to test in dev mode this.logger.debug(`Magic link: ${magicLink}`); } diff --git a/packages/backend/server/src/core/auth/service.ts b/packages/backend/server/src/core/auth/service.ts index e489e90333..c50273aa94 100644 --- a/packages/backend/server/src/core/auth/service.ts +++ b/packages/backend/server/src/core/auth/service.ts @@ -49,7 +49,7 @@ export class AuthService implements OnApplicationBootstrap { ) {} async onApplicationBootstrap() { - if (this.config.node.dev) { + if (env.dev) { await createDevUsers(this.models); } } @@ -59,10 +59,12 @@ export class AuthService implements OnApplicationBootstrap { } /** + * @deprecated + * * This is a test only helper to quickly signup a user, do not use in production */ async signUp(email: string, password: string): Promise { - if (!this.config.node.test) { + if (!env.testing) { throw new SignUpForbidden( 'sign up helper is forbidden for non-test environment' ); diff --git a/packages/backend/server/src/core/config/__tests__/service.spec.ts b/packages/backend/server/src/core/config/__tests__/service.spec.ts new file mode 100644 index 0000000000..5c0bcb4158 --- /dev/null +++ b/packages/backend/server/src/core/config/__tests__/service.spec.ts @@ -0,0 +1,136 @@ +import { faker } from '@faker-js/faker'; +import test from 'ava'; +import Sinon from 'sinon'; + +import { createModule } from '../../../__tests__/create-module'; +import { Mockers } from '../../../__tests__/mocks'; +import { Models } from '../../../models'; +import { ServerService } from '../service'; + +const module = await createModule({ + providers: [ServerService], +}); +const service = module.get(ServerService); +const user = await module.create(Mockers.User); +const models = module.get(Models); + +test.afterEach(async () => { + Sinon.reset(); +}); + +test.after.always(async () => { + await module.close(); +}); + +test('should update config', async t => { + const oldValue = service.config.server.externalUrl; + const newValue = faker.internet.url(); + await service.updateConfig(user.id, [ + { + module: 'server', + key: 'externalUrl', + value: newValue, + }, + ]); + + t.not(service.config.server.externalUrl, oldValue); + t.is(service.config.server.externalUrl, newValue); +}); + +test('should validate config before update', async t => { + await t.throwsAsync( + service.updateConfig(user.id, [ + { + module: 'server', + key: 'externalUrl', + value: 'invalid-url@some-domain.com', + }, + ]), + { + message: `Invalid config for module [server] with key [externalUrl] +Value: "invalid-url@some-domain.com" +Error: Invalid url`, + } + ); + + t.not(service.config.server.externalUrl, 'invalid-url'); + + await t.throwsAsync( + service.updateConfig(user.id, [ + { + module: 'auth', + key: 'unknown-key', + value: 'invalid-value', + }, + ]), + { + message: `Invalid config for module [auth] with unknown key [unknown-key]`, + } + ); + + // @ts-expect-error allow + t.is(service.config.auth['unknown-key'], undefined); +}); + +test('should emit config.init event', async t => { + await service.onApplicationBootstrap(); + const event = module.event.last('config.init'); + t.is(event.name, 'config.init'); + t.deepEqual(event.payload, { + config: service.config, + }); +}); + +test('should revalidate config', async t => { + const outdatedValue = service.config.server.externalUrl; + const newValue = faker.internet.url(); + + await models.appConfig.save(user.id, [ + { + key: 'server.externalUrl', + value: newValue, + }, + ]); + + await service.revalidateConfig(); + + t.not(service.config.server.externalUrl, outdatedValue); + t.is(service.config.server.externalUrl, newValue); +}); + +test('should emit config changed event', async t => { + const newUrl = faker.internet.url(); + + await service.updateConfig(user.id, [ + { + module: 'server', + key: 'externalUrl', + value: newUrl, + }, + { + module: 'auth', + key: 'allowSignup', + value: false, + }, + ]); + + const updates = { + server: { + externalUrl: newUrl, + }, + auth: { + allowSignup: false, + }, + }; + + t.true( + module.event.emit.calledOnceWith('config.changed', { + updates, + }) + ); + t.true( + module.event.broadcast.calledOnceWith('config.changed.broadcast', { + updates, + }) + ); +}); diff --git a/packages/backend/server/src/core/config/config.ts b/packages/backend/server/src/core/config/config.ts index aa3b2bb4c2..2a163bf6be 100644 --- a/packages/backend/server/src/core/config/config.ts +++ b/packages/backend/server/src/core/config/config.ts @@ -1,23 +1,64 @@ -import { defineRuntimeConfig, ModuleConfig } from '../../base/config'; +import { z } from 'zod'; + +import { defineModuleConfig } from '../../base'; export interface ServerFlags { earlyAccessControl: boolean; - syncClientVersionCheck: boolean; } -declare module '../../base/config' { - interface AppConfig { - flags: ModuleConfig; +declare global { + interface AppConfigSchema { + server: { + externalUrl: string; + https: boolean; + host: string; + port: number; + path: string; + name: string | undefined; + }; + flags: ServerFlags; } } -defineRuntimeConfig('flags', { +defineModuleConfig('server', { + name: { + desc: 'A recognizable name for the server. Will be shown when connected with AFFiNE Desktop.', + default: 'AFFiNE Cloud', + }, + externalUrl: { + desc: `Base url of AFFiNE server, used for generating external urls. +Default to be \`[server.protocol]://[server.host][:server.port]\` if not specified. + `, + default: 'http://localhost:3010', + env: 'AFFINE_SERVER_EXTERNAL_URL', + shape: z.string().url(), + }, + https: { + desc: 'Whether the server is hosted on a ssl enabled domain (https://).', + default: false, + env: ['AFFINE_SERVER_HTTPS', 'boolean'], + shape: z.boolean(), + }, + host: { + desc: 'Where the server get deployed(FQDN).', + default: 'localhost', + env: 'AFFINE_SERVER_HOST', + }, + port: { + desc: 'Which port the server will listen on.', + default: 3010, + env: ['AFFINE_SERVER_PORT', 'integer'], + }, + path: { + desc: 'Subpath where the server get deployed if there is.', + default: '', + env: 'AFFINE_SERVER_SUB_PATH', + }, +}); + +defineModuleConfig('flags', { earlyAccessControl: { desc: 'Only allow users with early access features to access the app', default: false, }, - syncClientVersionCheck: { - desc: 'Only allow client with exact the same version with server to establish sync connections', - default: false, - }, }); diff --git a/packages/backend/server/src/core/config/index.ts b/packages/backend/server/src/core/config/index.ts index 1ea38185b9..a71df806d4 100644 --- a/packages/backend/server/src/core/config/index.ts +++ b/packages/backend/server/src/core/config/index.ts @@ -3,24 +3,25 @@ import './config'; import { Module } from '@nestjs/common'; import { + AppConfigResolver, ServerConfigResolver, ServerFeatureConfigResolver, - ServerRuntimeConfigResolver, - ServerServiceConfigResolver, } from './resolver'; import { ServerService } from './service'; @Module({ - providers: [ - ServerService, - ServerConfigResolver, - ServerFeatureConfigResolver, - ServerRuntimeConfigResolver, - ServerServiceConfigResolver, - ], + providers: [ServerService], exports: [ServerService], }) export class ServerConfigModule {} +@Module({ + imports: [ServerConfigModule], + providers: [ + ServerConfigResolver, + ServerFeatureConfigResolver, + AppConfigResolver, + ], +}) +export class ServerConfigResolverModule {} export { ServerService }; -export { ADD_ENABLED_FEATURES } from './server-feature'; export { ServerFeature } from './types'; diff --git a/packages/backend/server/src/core/config/resolver.ts b/packages/backend/server/src/core/config/resolver.ts index a1f5d66b6c..a76fa0bb38 100644 --- a/packages/backend/server/src/core/config/resolver.ts +++ b/packages/backend/server/src/core/config/resolver.ts @@ -3,23 +3,21 @@ import { Args, Field, GraphQLISODateTime, + InputType, Mutation, ObjectType, Query, - registerEnumType, ResolveField, Resolver, } from '@nestjs/graphql'; -import { RuntimeConfig, RuntimeConfigType } from '@prisma/client'; import { GraphQLJSON, GraphQLJSONObject } from 'graphql-scalars'; -import { Config, Runtime, URLHelper } from '../../base'; +import { Config, URLHelper } from '../../base'; +import { Namespace } from '../../env'; import { Feature } from '../../models'; -import { Public } from '../auth'; +import { CurrentUser, Public } from '../auth'; import { Admin } from '../common'; import { AvailableUserFeatureConfig } from '../features'; -import { ServerFlags } from './config'; -import { ENABLED_FEATURES } from './server-feature'; import { ServerService } from './service'; import { ServerConfigType } from './types'; @@ -37,10 +35,6 @@ export class CredentialsRequirementType { password!: PasswordLimitsType; } -registerEnumType(RuntimeConfigType, { - name: 'RuntimeConfigType', -}); - @ObjectType() export class ReleaseVersionType { @Field() @@ -56,43 +50,11 @@ export class ReleaseVersionType { changelog!: string; } -const RELEASE_CHANNEL_MAP = new Map([ - ['dev', 'canary'], - ['beta', 'beta'], - ['production', 'stable'], +const RELEASE_CHANNEL_MAP = new Map([ + [Namespace.Dev, 'canary'], + [Namespace.Beta, 'beta'], + [Namespace.Production, 'stable'], ]); -@ObjectType() -export class ServerRuntimeConfigType implements Partial { - @Field() - id!: string; - - @Field() - module!: string; - - @Field() - key!: string; - - @Field() - description!: string; - - @Field(() => GraphQLJSON) - value!: any; - - @Field(() => RuntimeConfigType) - type!: RuntimeConfigType; - - @Field(() => GraphQLISODateTime) - updatedAt!: Date; -} - -@ObjectType() -export class ServerFlagsType implements ServerFlags { - @Field() - earlyAccessControl!: boolean; - - @Field() - syncClientVersionCheck!: boolean; -} @Resolver(() => ServerConfigType) export class ServerConfigResolver { @@ -100,7 +62,6 @@ export class ServerConfigResolver { constructor( private readonly config: Config, - private readonly runtime: Runtime, private readonly url: URLHelper, private readonly server: ServerService ) {} @@ -111,16 +72,19 @@ export class ServerConfigResolver { }) serverConfig(): ServerConfigType { return { - name: this.config.serverName, - version: this.config.version, + name: + this.config.server.name ?? + (env.selfhosted + ? 'AFFiNE Selfhosted Cloud' + : env.namespaces.canary + ? 'AFFiNE Canary Cloud' + : env.namespaces.beta + ? 'AFFiNE Beta Cloud' + : 'AFFiNE Cloud'), + version: env.version, baseUrl: this.url.home, - type: this.config.type, - // BACKWARD COMPATIBILITY - // the old flavors contains `selfhosted` but it actually not flavor but deployment type - // this field should be removed after frontend feature flags implemented - flavor: this.config.type, - features: Array.from(ENABLED_FEATURES), - enableTelemetry: this.config.metrics.telemetry.enabled, + type: env.DEPLOYMENT_TYPE, + features: this.server.features, }; } @@ -128,31 +92,14 @@ export class ServerConfigResolver { description: 'credentials requirement', }) async credentialsRequirement() { - const config = await this.runtime.fetchAll({ - 'auth/password.max': true, - 'auth/password.min': true, - }); - return { password: { - minLength: config['auth/password.min'], - maxLength: config['auth/password.max'], + minLength: this.config.auth.passwordRequirements.min, + maxLength: this.config.auth.passwordRequirements.max, }, }; } - @ResolveField(() => ServerFlagsType, { - description: 'server flags', - }) - async flags(): Promise { - const records = await this.runtime.list('flags'); - - return records.reduce((flags, record) => { - flags[record.key as keyof ServerFlagsType] = record.value as any; - return flags; - }, {} as ServerFlagsType); - } - @ResolveField(() => Boolean, { description: 'whether server has been initialized', }) @@ -161,10 +108,15 @@ export class ServerConfigResolver { } @ResolveField(() => ReleaseVersionType, { + nullable: true, description: 'fetch latest available upgradable release of server', }) async availableUpgrade(): Promise { - const channel = RELEASE_CHANNEL_MAP.get(this.config.AFFINE_ENV) ?? 'stable'; + if (!env.selfhosted) { + return null; + } + + const channel = RELEASE_CHANNEL_MAP.get(env.NAMESPACE) ?? 'stable'; const url = `https://affine.pro/api/worker/releases?channel=${channel}`; try { @@ -191,7 +143,7 @@ export class ServerConfigResolver { }>; const latest = releases.at(0); - if (!latest || latest.name === this.config.version) { + if (!latest || latest.name === env.version) { return null; } @@ -218,124 +170,38 @@ export class ServerFeatureConfigResolver extends AvailableUserFeatureConfig { } } -@ObjectType() -class ServerServiceConfig { +@InputType() +class UpdateAppConfigInput { @Field() - name!: string; + module!: string; - @Field(() => GraphQLJSONObject) - config!: any; -} + @Field() + key!: string; -interface ServerServeConfig { - https: boolean; - host: string; - port: number; - externalUrl: string; -} - -interface ServerMailerConfig { - host?: string | null; - port?: number | null; - secure?: boolean | null; - service?: string | null; - sender?: string | null; -} - -interface ServerDatabaseConfig { - host: string; - port: number; - user?: string | null; - database: string; + @Field(() => GraphQLJSON) + value!: any; } @Admin() -@Resolver(() => ServerRuntimeConfigType) -export class ServerRuntimeConfigResolver { - constructor(private readonly runtime: Runtime) {} +@Resolver(() => GraphQLJSONObject) +export class AppConfigResolver { + constructor(private readonly service: ServerService) {} - @Query(() => [ServerRuntimeConfigType], { - description: 'get all server runtime configurable settings', + @Query(() => GraphQLJSONObject, { + description: 'get the whole app configuration', }) - serverRuntimeConfig(): Promise { - return this.runtime.list(); + appConfig() { + return this.service.config; } - @Mutation(() => ServerRuntimeConfigType, { - description: 'update server runtime configurable setting', + @Mutation(() => GraphQLJSONObject, { + description: 'update app configuration', }) - async updateRuntimeConfig( - @Args('id') id: string, - @Args({ type: () => GraphQLJSON, name: 'value' }) value: any - ): Promise { - return await this.runtime.set(id as any, value); - } - - @Mutation(() => [ServerRuntimeConfigType], { - description: 'update multiple server runtime configurable settings', - }) - async updateRuntimeConfigs( - @Args({ type: () => GraphQLJSONObject, name: 'updates' }) updates: any - ): Promise { - const keys = Object.keys(updates); - const results = await Promise.all( - keys.map(key => this.runtime.set(key as any, updates[key])) - ); - - return results; - } -} - -@Admin() -@Resolver(() => ServerServiceConfig) -export class ServerServiceConfigResolver { - constructor(private readonly config: Config) {} - - @Query(() => [ServerServiceConfig]) - serverServiceConfigs() { - return [ - { - name: 'server', - config: this.serve(), - }, - { - name: 'mailer', - config: this.mail(), - }, - { - name: 'database', - config: this.database(), - }, - ]; - } - - serve(): ServerServeConfig { - return this.config.server; - } - - mail(): ServerMailerConfig { - const sender = - typeof this.config.mailer.from === 'string' - ? this.config.mailer.from - : this.config.mailer.from?.address; - - return { - host: this.config.mailer.host, - port: this.config.mailer.port, - secure: this.config.mailer.secure, - service: this.config.mailer.service, - sender, - }; - } - - database(): ServerDatabaseConfig { - const url = new URL(this.config.prisma.datasourceUrl); - - return { - host: url.hostname, - port: Number(url.port), - user: url.username, - database: url.pathname.slice(1) ?? url.username, - }; + async updateAppConfig( + @CurrentUser() me: CurrentUser, + @Args('updates', { type: () => [UpdateAppConfigInput] }) + updates: UpdateAppConfigInput[] + ): Promise> { + return await this.service.updateConfig(me.id, updates); } } diff --git a/packages/backend/server/src/core/config/server-feature.ts b/packages/backend/server/src/core/config/server-feature.ts deleted file mode 100644 index 44375bb914..0000000000 --- a/packages/backend/server/src/core/config/server-feature.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { ServerFeature } from './types'; - -export const ENABLED_FEATURES: Set = new Set(); -export function ADD_ENABLED_FEATURES(feature: ServerFeature) { - ENABLED_FEATURES.add(feature); -} -export { ServerFeature }; diff --git a/packages/backend/server/src/core/config/service.ts b/packages/backend/server/src/core/config/service.ts index 810d62cfbf..1ea543ac3a 100644 --- a/packages/backend/server/src/core/config/service.ts +++ b/packages/backend/server/src/core/config/service.ts @@ -1,17 +1,120 @@ -import { Injectable } from '@nestjs/common'; -import { PrismaClient } from '@prisma/client'; +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { set } from 'lodash-es'; + +import { ConfigFactory, EventBus, OnEvent } from '../../base'; +import { Models } from '../../models'; +import { ServerFeature } from './types'; + +declare global { + interface Events { + 'config.init': { + config: DeepReadonly; + }; + 'config.changed': { + updates: DeepPartial; + }; + 'config.changed.broadcast': { + updates: DeepPartial; + }; + } +} @Injectable() -export class ServerService { +export class ServerService implements OnApplicationBootstrap { private _initialized: boolean | null = null; - constructor(private readonly db: PrismaClient) {} + readonly #features = new Set(); + readonly #logger = new Logger(ServerService.name); + + constructor( + private readonly models: Models, + private readonly configFactory: ConfigFactory, + private readonly event: EventBus + ) {} + + async onApplicationBootstrap() { + await this.setup(); + } + + get features() { + return Array.from(this.#features); + } async initialized() { if (!this._initialized) { - const userCount = await this.db.user.count(); + const userCount = await this.models.user.count(); this._initialized = userCount > 0; } return this._initialized; } + + enableFeature(feature: ServerFeature) { + this.#features.add(feature); + } + + disableFeature(feature: ServerFeature) { + this.#features.delete(feature); + } + + get config() { + return this.configFactory.config; + } + + async updateConfig( + user: string, + updates: Array<{ module: string; key: string; value: any }> + ): Promise> { + this.configFactory.validate(updates); + + const promises = await this.models.appConfig.save( + user, + updates.map(update => ({ + key: `${update.module}.${update.key}`, + value: update.value, + })) + ); + + const overrides: DeepPartial = {}; + // only take successfully saved configs + promises.forEach(promise => { + if (promise.status === 'fulfilled') { + set(overrides, promise.value.id, promise.value.value); + } else { + this.#logger.error(`Failed to save app config`, promise.reason); + } + }); + this.configFactory.override(overrides); + this.event.emit('config.changed', { updates: overrides }); + this.event.broadcast('config.changed.broadcast', { updates: overrides }); + return overrides; + } + + @OnEvent('config.changed.broadcast') + onConfigChangedBroadcast(updates: DeepPartial) { + this.configFactory.override(updates); + this.event.emit('config.changed', { updates }); + } + + async revalidateConfig() { + const overrides = await this.loadDbOverrides(); + this.configFactory.override(overrides); + this.event.emit('config.changed', { updates: overrides }); + } + + private async setup() { + const overrides = await this.loadDbOverrides(); + this.configFactory.override(overrides); + this.event.emit('config.init', { config: this.configFactory.config }); + } + + private async loadDbOverrides() { + const configs = await this.models.appConfig.load(); + const overrides: DeepPartial = {}; + + configs.forEach(config => { + set(overrides, config.id, config.value); + }); + + return overrides; + } } diff --git a/packages/backend/server/src/core/config/types.ts b/packages/backend/server/src/core/config/types.ts index 8e120ebe6b..643cc97b5e 100644 --- a/packages/backend/server/src/core/config/types.ts +++ b/packages/backend/server/src/core/config/types.ts @@ -1,6 +1,6 @@ import { Field, ObjectType, registerEnumType } from '@nestjs/graphql'; -import { DeploymentType } from '../../base'; +import { DeploymentType } from '../../env'; export enum ServerFeature { Captcha = 'captcha', @@ -34,15 +34,6 @@ export class ServerConfigType { @Field(() => DeploymentType, { description: 'server type' }) type!: DeploymentType; - /** - * @deprecated - */ - @Field({ description: 'server flavor', deprecationReason: 'use `features`' }) - flavor!: string; - @Field(() => [ServerFeature], { description: 'enabled server features' }) features!: ServerFeature[]; - - @Field({ description: 'enable telemetry' }) - enableTelemetry!: boolean; } diff --git a/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts index adee142c53..a88dd3bccf 100644 --- a/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts +++ b/packages/backend/server/src/core/doc-renderer/__tests__/controller.spec.ts @@ -5,36 +5,23 @@ import ava, { TestFn } from 'ava'; import { Doc as YDoc } from 'yjs'; import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { AppModule } from '../../../app.module'; -import { Config } from '../../../base'; -import { ConfigModule } from '../../../base/config'; +import { ConfigFactory } from '../../../base'; +import { Flavor } from '../../../env'; import { Models } from '../../../models'; import { PgWorkspaceDocStorageAdapter } from '../../doc'; const test = ava as TestFn<{ models: Models; app: TestingApp; - config: Config; adapter: PgWorkspaceDocStorageAdapter; }>; test.before(async t => { - const app = await createTestingApp({ - imports: [ - ConfigModule.forRoot({ - flavor: { - doc: false, - }, - docService: { - endpoint: '', - }, - }), - AppModule, - ], - }); + // @ts-expect-error testing + env.FLAVOR = Flavor.Renderer; + const app = await createTestingApp(); t.context.models = app.get(Models); - t.context.config = app.get(Config); t.context.adapter = app.get(PgWorkspaceDocStorageAdapter); t.context.app = app; }); @@ -43,7 +30,11 @@ let user: User; let workspace: Workspace; test.beforeEach(async t => { - t.context.config.docService.endpoint = t.context.app.url(); + t.context.app.get(ConfigFactory).override({ + docService: { + endpoint: t.context.app.url(), + }, + }); await t.context.app.initTestingDB(); user = await t.context.models.user.create({ email: 'test@affine.pro', diff --git a/packages/backend/server/src/core/doc-renderer/controller.ts b/packages/backend/server/src/core/doc-renderer/controller.ts index f7a1b84ad1..d1abff0514 100644 --- a/packages/backend/server/src/core/doc-renderer/controller.ts +++ b/packages/backend/server/src/core/doc-renderer/controller.ts @@ -5,7 +5,7 @@ import { Controller, Get, Logger, Req, Res } from '@nestjs/common'; import type { Request, Response } from 'express'; import isMobile from 'is-mobile'; -import { Config, metrics } from '../../base'; +import { metrics } from '../../base'; import { Models } from '../../models'; import { htmlSanitize } from '../../native'; import { Public } from '../auth'; @@ -51,14 +51,11 @@ export class DocRendererController { constructor( private readonly doc: DocReader, - private readonly config: Config, private readonly models: Models ) { - this.webAssets = this.readHtmlAssets( - join(this.config.projectRoot, 'static') - ); + this.webAssets = this.readHtmlAssets(join(env.projectRoot, 'static')); this.mobileAssets = this.readHtmlAssets( - join(this.config.projectRoot, 'static/mobile') + join(env.projectRoot, 'static/mobile') ); } @@ -66,7 +63,7 @@ export class DocRendererController { @Get('/*') async render(@Req() req: Request, @Res() res: Response) { const assets: HtmlAssets = - this.config.affine.canary && + env.namespaces.canary && isMobile({ ua: req.headers['user-agent'] ?? undefined, }) @@ -141,13 +138,13 @@ export class DocRendererController { // @TODO(@forehalo): pre-compile html template to accelerate serializing _render(opts: RenderOptions | null, assets: HtmlAssets): string { // TODO(@forehalo): how can we enable the type reference to @affine/env - const env: Record = { + const envMeta: Record = { publicPath: assets.publicPath, renderer: 'ssr', }; - if (this.config.isSelfhosted) { - env.isSelfHosted = true; + if (env.selfhosted) { + envMeta.isSelfHosted = true; } const title = opts?.title @@ -192,7 +189,7 @@ export class DocRendererController { - ${Object.entries(env) + ${Object.entries(envMeta) .map(([key, val]) => ``) .join('\n')} ${assets.css.map(url => ``).join('\n')} @@ -216,7 +213,7 @@ export class DocRendererController { readFileSync(manifestPath, 'utf-8') ); - const publicPath = this.config.isSelfhosted ? '/' : assets.publicPath; + const publicPath = env.selfhosted ? '/' : assets.publicPath; assets.publicPath = publicPath; assets.js = assets.js.map(path => publicPath + path); @@ -224,7 +221,7 @@ export class DocRendererController { return assets; } catch (e) { - if (this.config.node.prod) { + if (env.prod) { throw e; } else { return defaultAssets; diff --git a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts index b9923d4a52..40216f797b 100644 --- a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts +++ b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts @@ -5,9 +5,7 @@ import { User, Workspace } from '@prisma/client'; import ava, { TestFn } from 'ava'; import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { AppModule } from '../../../app.module'; import { CryptoHelper } from '../../../base'; -import { ConfigModule } from '../../../base/config'; import { Models } from '../../../models'; import { DatabaseDocReader } from '../../doc'; @@ -19,9 +17,7 @@ const test = ava as TestFn<{ }>; test.before(async t => { - const app = await createTestingApp({ - imports: [ConfigModule.forRoot(), AppModule], - }); + const app = await createTestingApp(); t.context.models = app.get(Models); t.context.crypto = app.get(CryptoHelper); diff --git a/packages/backend/server/src/core/doc-service/config.ts b/packages/backend/server/src/core/doc-service/config.ts index 64dcf38345..9eb4e3136b 100644 --- a/packages/backend/server/src/core/doc-service/config.ts +++ b/packages/backend/server/src/core/doc-service/config.ts @@ -1,19 +1,16 @@ -import { defineStartupConfig, ModuleConfig } from '../../base/config'; +import { defineModuleConfig } from '../../base'; -interface DocServiceStartupConfigurations { - /** - * The endpoint of the doc service. - * Example: http://doc-service:3020 - */ - endpoint: string; -} - -declare module '../../base/config' { - interface AppConfig { - docService: ModuleConfig; +declare global { + interface AppConfigSchema { + docService: { + endpoint: string; + }; } } -defineStartupConfig('docService', { - endpoint: '', +defineModuleConfig('docService', { + endpoint: { + desc: 'The endpoint of the doc service.', + default: '', + }, }); diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts index baafb8f0dc..a4d7385349 100644 --- a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts +++ b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts @@ -6,8 +6,6 @@ import ava, { TestFn } from 'ava'; import { applyUpdate, Doc as YDoc } from 'yjs'; import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { AppModule } from '../../../app.module'; -import { ConfigModule } from '../../../base/config'; import { Models } from '../../../models'; import { WorkspaceBlobStorage } from '../../storage/wrappers/blob'; import { DocReader, PgWorkspaceDocStorageAdapter } from '..'; @@ -22,9 +20,7 @@ const test = ava as TestFn<{ }>; test.before(async t => { - const app = await createTestingApp({ - imports: [ConfigModule.forRoot(), AppModule], - }); + const app = await createTestingApp(); t.context.models = app.get(Models); t.context.docReader = app.get(DocReader); diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts index a3d9de32d2..9f7eb87734 100644 --- a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts +++ b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts @@ -6,9 +6,8 @@ import ava, { TestFn } from 'ava'; import { applyUpdate, Doc as YDoc } from 'yjs'; import { createTestingApp, type TestingApp } from '../../../__tests__/utils'; -import { AppModule } from '../../../app.module'; -import { Config, UserFriendlyError } from '../../../base'; -import { ConfigModule } from '../../../base/config'; +import { UserFriendlyError } from '../../../base'; +import { ConfigFactory } from '../../../base/config'; import { Models } from '../../../models'; import { DatabaseDocReader, DocReader, PgWorkspaceDocStorageAdapter } from '..'; import { RpcDocReader } from '../reader'; @@ -16,40 +15,45 @@ import { RpcDocReader } from '../reader'; const test = ava as TestFn<{ models: Models; app: TestingApp; + docApp: TestingApp; docReader: DocReader; databaseDocReader: DatabaseDocReader; adapter: PgWorkspaceDocStorageAdapter; - config: Config; + config: ConfigFactory; }>; test.before(async t => { - const app = await createTestingApp({ - imports: [ - ConfigModule.forRoot({ - flavor: { - doc: false, - }, - docService: { - endpoint: '', - }, - }), - AppModule, - ], - }); + // test key + process.env.AFFINE_PRIVATE_KEY = `-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgS3IAkshQuSmFWGpe +rGTg2vwaC3LdcvBQlYHHMBYJZMyhRANCAAQXdT/TAh4neNEpd4UqpDIEqWv0XvFo +BRJxGsC5I/fetqObdx1+KEjcm8zFU2xLaUTw9IZCu8OslloOjQv4ur0a +-----END PRIVATE KEY-----`; + // @ts-expect-error testing + env.FLAVOR = 'renderer'; + const notDocApp = await createTestingApp(); + // @ts-expect-error testing + env.FLAVOR = 'doc'; + const docApp = await createTestingApp(); - t.context.models = app.get(Models); - t.context.docReader = app.get(DocReader); - t.context.databaseDocReader = app.get(DatabaseDocReader); - t.context.adapter = app.get(PgWorkspaceDocStorageAdapter); - t.context.config = app.get(Config); - t.context.app = app; + t.context.models = notDocApp.get(Models); + t.context.docReader = notDocApp.get(DocReader); + t.context.databaseDocReader = docApp.get(DatabaseDocReader); + t.context.adapter = docApp.get(PgWorkspaceDocStorageAdapter); + t.context.config = notDocApp.get(ConfigFactory); + t.context.app = notDocApp; + t.context.docApp = docApp; }); let user: User; let workspace: Workspace; test.beforeEach(async t => { - t.context.config.docService.endpoint = t.context.app.url(); + t.context.config.override({ + docService: { + endpoint: t.context.docApp.url(), + }, + }); await t.context.app.initTestingDB(); user = await t.context.models.user.create({ email: 'test@affine.pro', @@ -63,6 +67,7 @@ test.afterEach.always(() => { test.after.always(async t => { await t.context.app.close(); + await t.context.docApp.close(); }); test('should return null when doc not found', async t => { @@ -113,7 +118,11 @@ test('should throw error when doc service internal error', async t => { test('should fallback to database doc reader when endpoint network error', async t => { const { docReader } = t.context; - t.context.config.docService.endpoint = 'http://localhost:13010'; + t.context.config.override({ + docService: { + endpoint: 'http://localhost:13010', + }, + }); const docId = randomUUID(); const timestamp = Date.now(); await t.context.models.doc.createUpdates([ @@ -223,7 +232,11 @@ test('should return doc diff', async t => { test('should get doc diff fallback to database doc reader when endpoint network error', async t => { const { docReader } = t.context; - t.context.config.docService.endpoint = 'http://localhost:13010'; + t.context.config.override({ + docService: { + endpoint: 'http://localhost:13010', + }, + }); const docId = randomUUID(); const timestamp = Date.now(); let updates: Buffer[] = []; diff --git a/packages/backend/server/src/core/doc/config.ts b/packages/backend/server/src/core/doc/config.ts index 1add392c4f..ca61f34d00 100644 --- a/packages/backend/server/src/core/doc/config.ts +++ b/packages/backend/server/src/core/doc/config.ts @@ -1,44 +1,25 @@ -import { - defineRuntimeConfig, - defineStartupConfig, - ModuleConfig, -} from '../../base/config'; +import { defineModuleConfig } from '../../base'; -interface DocStartupConfigurations { - history: { - /** - * How long the buffer time of creating a new history snapshot when doc get updated. - * - * in {ms} - */ - interval: number; - }; -} - -interface DocRuntimeConfigurations { - /** - * Use `y-octo` to merge updates at the same time when merging using Yjs. - * - * This is an experimental feature, and aimed to check the correctness of JwstCodec. - */ - experimentalMergeWithYOcto: boolean; -} - -declare module '../../base/config' { - interface AppConfig { - doc: ModuleConfig; +declare global { + interface AppConfigSchema { + doc: { + history: { + interval: number; + }; + experimental: { + yocto: boolean; + }; + }; } } -defineStartupConfig('doc', { - history: { - interval: 1000 * 60 * 10 /* 10 mins */, - }, -}); - -defineRuntimeConfig('doc', { - experimentalMergeWithYOcto: { +defineModuleConfig('doc', { + 'experimental.yocto': { desc: 'Use `y-octo` to merge updates at the same time when merging using Yjs.', default: false, }, + 'history.interval': { + desc: 'The minimum time interval in milliseconds of creating a new history snapshot when doc get updated.', + default: 1000 * 60 * 10 /* 10 mins */, + }, }); diff --git a/packages/backend/server/src/core/doc/options.ts b/packages/backend/server/src/core/doc/options.ts index 4793a1cbf9..75e7879f94 100644 --- a/packages/backend/server/src/core/doc/options.ts +++ b/packages/backend/server/src/core/doc/options.ts @@ -2,13 +2,8 @@ import { Injectable, Logger } from '@nestjs/common'; import { chunk } from 'lodash-es'; import * as Y from 'yjs'; -import { - CallMetric, - Config, - mergeUpdatesInApplyWay as yotcoMergeUpdates, - metrics, - Runtime, -} from '../../base'; +import { CallMetric, Config, metrics } from '../../base'; +import { mergeUpdatesInApplyWay as yoctoMergeUpdates } from '../../native'; import { QuotaService } from '../quota'; import { DocStorageOptions as IDocStorageOptions } from './storage'; @@ -35,7 +30,6 @@ export class DocStorageOptions implements IDocStorageOptions { constructor( private readonly config: Config, - private readonly runtime: Runtime, private readonly quota: QuotaService ) {} @@ -43,19 +37,17 @@ export class DocStorageOptions implements IDocStorageOptions { const doc = await this.recoverDoc(updates); const yjsResult = Buffer.from(Y.encodeStateAsUpdate(doc)); - const useYocto = await this.runtime.fetch('doc/experimentalMergeWithYOcto'); - - if (useYocto) { + if (this.config.doc.experimental.yocto) { metrics.jwst.counter('codec_merge_counter').add(1); let log = false; let yoctoResult: Buffer | null = null; try { - yoctoResult = yotcoMergeUpdates(updates.map(Buffer.from)); + yoctoResult = yoctoMergeUpdates(updates.map(Buffer.from)); if (!compare(yjsResult, yoctoResult)) { metrics.jwst.counter('codec_not_match').add(1); this.logger.warn(`yocto codec result doesn't match yjs codec result`); log = true; - if (this.config.node.dev) { + if (env.dev) { this.logger.warn(`Expected:\n ${yjsResult.toString('hex')}`); this.logger.warn(`Result:\n ${yoctoResult.toString('hex')}`); } @@ -66,14 +58,14 @@ export class DocStorageOptions implements IDocStorageOptions { log = true; } - if (log && this.config.node.dev) { + if (log && env.dev) { this.logger.warn( `Updates: ${updates.map(u => Buffer.from(u).toString('hex')).join('\n')}` ); } if ( - this.config.affine.canary && + env.namespaces.canary && yoctoResult && yoctoResult.length > 2 /* simple test for non-empty yjs binary */ ) { diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.ts index d35fe7cd8b..05124b89bd 100644 --- a/packages/backend/server/src/core/doc/reader.ts +++ b/packages/backend/server/src/core/doc/reader.ts @@ -402,11 +402,11 @@ export class RpcDocReader extends DatabaseDocReader { export const DocReaderProvider: FactoryProvider = { provide: DocReader, - useFactory: (config: Config, ref: ModuleRef) => { - if (config.flavor.doc) { + useFactory: (ref: ModuleRef) => { + if (env.flavors.doc) { return ref.create(DatabaseDocReader); } return ref.create(RpcDocReader); }, - inject: [Config, ModuleRef], + inject: [ModuleRef], }; diff --git a/packages/backend/server/src/core/features/service.ts b/packages/backend/server/src/core/features/service.ts index deb9879395..2452a52c0a 100644 --- a/packages/backend/server/src/core/features/service.ts +++ b/packages/backend/server/src/core/features/service.ts @@ -1,6 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; -import { Runtime } from '../../base'; +import { Config } from '../../base'; import { Models } from '../../models'; const STAFF = ['@toeverything.info', '@affine.pro']; @@ -15,8 +15,8 @@ export class FeatureService { protected logger = new Logger(FeatureService.name); constructor( - private readonly models: Models, - private readonly runtime: Runtime + private readonly config: Config, + private readonly models: Models ) {} // ======== Admin ======== @@ -73,9 +73,7 @@ export class FeatureService { email: string, type: EarlyAccessType = EarlyAccessType.App ) { - const earlyAccessControlEnabled = await this.runtime.fetch( - 'flags/earlyAccessControl' - ); + const earlyAccessControlEnabled = this.config.flags.earlyAccessControl; if (earlyAccessControlEnabled && !this.isStaff(email)) { const user = await this.models.user.getUserByEmail(email); diff --git a/packages/backend/server/src/core/features/types.ts b/packages/backend/server/src/core/features/types.ts index 08db922999..9405109fd5 100644 --- a/packages/backend/server/src/core/features/types.ts +++ b/packages/backend/server/src/core/features/types.ts @@ -1,12 +1,9 @@ -import { Inject, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; -import { Config } from '../../base'; import { Feature, UserFeatureName } from '../../models'; @Injectable() export class AvailableUserFeatureConfig { - @Inject(Config) private readonly config!: Config; - availableUserFeatures(): Set { return new Set([ Feature.Admin, @@ -18,7 +15,7 @@ export class AvailableUserFeatureConfig { configurableUserFeatures(): Set { return new Set( - this.config.isSelfhosted + env.selfhosted ? [Feature.Admin, Feature.UnlimitedCopilot] : [ Feature.EarlyAccess, diff --git a/packages/backend/server/src/core/index.ts b/packages/backend/server/src/core/index.ts new file mode 100644 index 0000000000..f03c2281a9 --- /dev/null +++ b/packages/backend/server/src/core/index.ts @@ -0,0 +1 @@ +export * from './config'; diff --git a/packages/backend/server/src/core/mail/config.ts b/packages/backend/server/src/core/mail/config.ts index b2a2754512..a77dd28c52 100644 --- a/packages/backend/server/src/core/mail/config.ts +++ b/packages/backend/server/src/core/mail/config.ts @@ -1,16 +1,54 @@ -import SMTPTransport from 'nodemailer/lib/smtp-transport'; +import { defineModuleConfig } from '../../base'; -import { defineStartupConfig, ModuleConfig } from '../../base/config'; - -declare module '../../base/config' { - interface AppConfig { - /** - * Configurations for mail service used to post auth or bussiness mails. - * - * @see https://nodemailer.com/smtp/ - */ - mailer: ModuleConfig; +declare global { + interface AppConfigSchema { + mailer: { + enabled: boolean; + SMTP: { + host: string; + port: number; + username: string; + password: string; + ignoreTLS: boolean; + sender: string; + }; + }; } } -defineStartupConfig('mailer', {}); +defineModuleConfig('mailer', { + enabled: { + desc: 'Whether enabled mail service.', + default: false, + }, + 'SMTP.host': { + desc: 'Host of the email server (e.g. smtp.gmail.com)', + default: '', + env: 'MAILER_HOST', + }, + 'SMTP.port': { + desc: 'Port of the email server (they commonly are 25, 465 or 587)', + default: 465, + env: ['MAILER_PORT', 'integer'], + }, + 'SMTP.username': { + desc: 'Username used to authenticate the email server', + default: '', + env: 'MAILER_USER', + }, + 'SMTP.password': { + desc: 'Password used to authenticate the email server', + default: '', + env: 'MAILER_PASSWORD', + }, + 'SMTP.sender': { + desc: 'Sender of all the emails (e.g. "AFFiNE Team ")', + default: '', + env: 'MAILER_SENDER', + }, + 'SMTP.ignoreTLS': { + desc: "Whether ignore email server's TSL certification verification. Enable it for self-signed certificates.", + default: false, + env: 'MAILER_IGNORE_TLS', + }, +}); diff --git a/packages/backend/server/src/core/mail/sender.ts b/packages/backend/server/src/core/mail/sender.ts index e07ab5e07d..fc3beba6b6 100644 --- a/packages/backend/server/src/core/mail/sender.ts +++ b/packages/backend/server/src/core/mail/sender.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { createTestAccount, createTransport, @@ -8,7 +8,7 @@ import { } from 'nodemailer'; import SMTPTransport from 'nodemailer/lib/smtp-transport'; -import { Config, metrics } from '../../base'; +import { Config, metrics, OnEvent } from '../../base'; export type SendOptions = Omit & { to: string; @@ -17,25 +17,51 @@ export type SendOptions = Omit & { }; @Injectable() -export class MailSender implements OnModuleInit { +export class MailSender { private readonly logger = new Logger(MailSender.name); private smtp: Transporter | null = null; private usingTestAccount = false; constructor(private readonly config: Config) {} - onModuleInit() { - this.createSMTP(this.config.mailer); + @OnEvent('config.init') + onConfigInit() { + this.setup(); } - createSMTP(config: SMTPTransport.Options) { - if (config.host) { - this.smtp = createTransport(config); - } else if (this.config.node.dev) { + @OnEvent('config.changed') + onConfigChanged(event: Events['config.changed']) { + if ('mailer' in event.updates) { + this.setup(); + } + } + + private setup() { + const { SMTP, enabled } = this.config.mailer; + + if (!enabled) { + this.smtp = null; + return; + } + + const opts: SMTPTransport.Options = { + host: SMTP.host, + port: SMTP.port, + tls: { + rejectUnauthorized: !SMTP.ignoreTLS, + }, + auth: { + user: SMTP.username, + pass: SMTP.password, + }, + }; + + if (SMTP.host) { + this.smtp = createTransport(opts); + } else if (env.dev) { createTestAccount((err, account) => { if (!err) { this.smtp = createTransport({ - from: 'noreply@toeverything.info', - ...this.config.mailer, + ...opts, ...account.smtp, auth: { user: account.user, @@ -59,7 +85,7 @@ export class MailSender implements OnModuleInit { metrics.mail.counter('send_total').add(1, { name }); try { const result = await this.smtp.sendMail({ - from: this.config.mailer.from, + from: this.config.mailer.SMTP.sender, ...options, }); diff --git a/packages/backend/server/src/core/notification/service.ts b/packages/backend/server/src/core/notification/service.ts index a785f3202a..8b80462eea 100644 --- a/packages/backend/server/src/core/notification/service.ts +++ b/packages/backend/server/src/core/notification/service.ts @@ -1,12 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; -import { - Config, - NotificationNotFound, - PaginationInput, - URLHelper, -} from '../../base'; +import { NotificationNotFound, PaginationInput, URLHelper } from '../../base'; import { DEFAULT_WORKSPACE_NAME, InvitationNotificationCreate, @@ -30,8 +25,7 @@ export class NotificationService { private readonly models: Models, private readonly docReader: DocReader, private readonly mailer: Mailer, - private readonly url: URLHelper, - private readonly config: Config + private readonly url: URLHelper ) {} async cleanExpiredNotifications() { @@ -100,7 +94,7 @@ export class NotificationService { private async sendInvitationEmail(input: InvitationNotificationCreate) { const inviteUrl = this.url.link(`/invite/${input.body.inviteId}`); - if (this.config.node.dev) { + if (env.dev) { // make it easier to test in dev mode this.logger.debug(`Invite link: ${inviteUrl}`); } diff --git a/packages/backend/server/src/core/selfhost/controller.ts b/packages/backend/server/src/core/selfhost/controller.ts index 35ba8a3136..19af802d63 100644 --- a/packages/backend/server/src/core/selfhost/controller.ts +++ b/packages/backend/server/src/core/selfhost/controller.ts @@ -3,10 +3,11 @@ import type { Request, Response } from 'express'; import { ActionForbidden, + Config, InternalServerError, Mutex, PasswordRequired, - Runtime, + UseNamedGuard, } from '../../base'; import { Models } from '../../models'; import { AuthService, Public } from '../auth'; @@ -18,14 +19,15 @@ interface CreateUserInput { password: string; } +@UseNamedGuard('selfhost') @Controller('/api/setup') export class CustomSetupController { constructor( + private readonly config: Config, private readonly models: Models, private readonly auth: AuthService, private readonly mutex: Mutex, - private readonly server: ServerService, - private readonly runtime: Runtime + private readonly server: ServerService ) {} @Public() @@ -45,15 +47,10 @@ export class CustomSetupController { throw new PasswordRequired(); } - const config = await this.runtime.fetchAll({ - 'auth/password.max': true, - 'auth/password.min': true, - }); - - validators.assertValidPassword(input.password, { - max: config['auth/password.max'], - min: config['auth/password.min'], - }); + validators.assertValidPassword( + input.password, + this.config.auth.passwordRequirements + ); await using lock = await this.mutex.acquire('createFirstAdmin'); diff --git a/packages/backend/server/src/core/selfhost/guard.ts b/packages/backend/server/src/core/selfhost/guard.ts new file mode 100644 index 0000000000..dd673f3ed5 --- /dev/null +++ b/packages/backend/server/src/core/selfhost/guard.ts @@ -0,0 +1,18 @@ +import { Injectable } from '@nestjs/common'; + +import { GuardProvider } from '../../base/guard'; + +declare module '../../base/guard' { + interface RegisterGuardName { + selfhost: 'selfhost'; + } +} + +@Injectable() +export class SelfhostGuard extends GuardProvider { + override name = 'selfhost' as const; + + override canActivate() { + return env.selfhosted; + } +} diff --git a/packages/backend/server/src/core/selfhost/index.ts b/packages/backend/server/src/core/selfhost/index.ts index 11282bac9a..1c8dfb6204 100644 --- a/packages/backend/server/src/core/selfhost/index.ts +++ b/packages/backend/server/src/core/selfhost/index.ts @@ -4,12 +4,13 @@ import { AuthModule } from '../auth'; import { ServerConfigModule } from '../config'; import { UserModule } from '../user'; import { CustomSetupController } from './controller'; +import { SelfhostGuard } from './guard'; import { SetupMiddleware } from './setup'; import { StaticFilesResolver } from './static'; @Module({ imports: [AuthModule, UserModule, ServerConfigModule], - providers: [SetupMiddleware, StaticFilesResolver], + providers: [SetupMiddleware, StaticFilesResolver, SelfhostGuard], controllers: [CustomSetupController], }) export class SelfhostModule {} diff --git a/packages/backend/server/src/core/selfhost/static.ts b/packages/backend/server/src/core/selfhost/static.ts index d1f4c1ef24..84be599033 100644 --- a/packages/backend/server/src/core/selfhost/static.ts +++ b/packages/backend/server/src/core/selfhost/static.ts @@ -26,7 +26,7 @@ export class StaticFilesResolver implements OnModuleInit { const app = this.adapterHost.httpAdapter.getInstance(); // for example, '/affine' in host [//host.com/affine] const basePath = this.config.server.path; - const staticPath = join(this.config.projectRoot, 'static'); + const staticPath = join(env.projectRoot, 'static'); // web => { // affine: 'static/index.html', @@ -69,7 +69,7 @@ export class StaticFilesResolver implements OnModuleInit { join( staticPath, 'admin', - this.config.isSelfhosted ? 'selfhost.html' : 'index.html' + env.selfhosted ? 'selfhost.html' : 'index.html' ) ); } @@ -107,7 +107,7 @@ export class StaticFilesResolver implements OnModuleInit { // fallback all unknown routes app.get([basePath, basePath + '/*'], this.check.use, (req, res) => { const mobile = - this.config.affine.canary && + env.namespaces.canary && isMobile({ ua: req.headers['user-agent'] ?? undefined, }); @@ -116,7 +116,7 @@ export class StaticFilesResolver implements OnModuleInit { join( staticPath, mobile ? 'mobile' : '', - this.config.isSelfhosted ? 'selfhost.html' : 'index.html' + env.selfhosted ? 'selfhost.html' : 'index.html' ) ); }); diff --git a/packages/backend/server/src/core/storage/config.ts b/packages/backend/server/src/core/storage/config.ts index 70301c0600..48caf37e83 100644 --- a/packages/backend/server/src/core/storage/config.ts +++ b/packages/backend/server/src/core/storage/config.ts @@ -1,34 +1,50 @@ -import { defineStartupConfig, ModuleConfig } from '../../base/config'; -import { StorageProviderType } from '../../base/storage'; +import { + defineModuleConfig, + StorageJSONSchema, + StorageProviderConfig, +} from '../../base'; -export type StorageConfig = { - provider: StorageProviderType; - bucket: string; -} & Ext; - -export interface StorageStartupConfigurations { - avatar: StorageConfig<{ - publicLinkFactory: (key: string) => string; - keyInPublicLink: (link: string) => string; - }>; - blob: StorageConfig; +export interface Storages { + avatar: { + storage: ConfigItem; + publicPath: string; + }; + blob: { + storage: ConfigItem; + }; } -declare module '../../base/config' { - interface AppConfig { - storages: ModuleConfig; +declare global { + interface AppConfigSchema { + storages: Storages; } } -defineStartupConfig('storages', { - avatar: { - provider: 'fs', - bucket: 'avatars', - publicLinkFactory: key => `/api/avatars/${key}`, - keyInPublicLink: link => link.split('/').pop() as string, +defineModuleConfig('storages', { + 'avatar.publicPath': { + desc: 'The public accessible path prefix for user avatars.', + default: '/api/avatars/', }, - blob: { - provider: 'fs', - bucket: 'blobs', + 'avatar.storage': { + desc: 'The config of storage for user avatars.', + default: { + provider: 'fs', + bucket: 'avatars', + config: { + path: '~/.affine/storage', + }, + }, + schema: StorageJSONSchema, + }, + 'blob.storage': { + desc: 'The config of storage for all uploaded blobs(images, videos, etc.).', + default: { + provider: 'fs', + bucket: 'blobs', + config: { + path: '~/.affine/storage', + }, + }, + schema: StorageJSONSchema, }, }); diff --git a/packages/backend/server/src/core/storage/wrappers/avatar.ts b/packages/backend/server/src/core/storage/wrappers/avatar.ts index ca4e9f3c81..388c1f0246 100644 --- a/packages/backend/server/src/core/storage/wrappers/avatar.ts +++ b/packages/backend/server/src/core/storage/wrappers/avatar.ts @@ -14,21 +14,23 @@ import { @Injectable() export class AvatarStorage { - public readonly provider: StorageProvider; - private readonly storageConfig: Config['storages']['avatar']; + private provider: StorageProvider; + + get config() { + return this.AFFiNEConfig.storages.avatar; + } constructor( - private readonly config: Config, + private readonly AFFiNEConfig: Config, private readonly url: URLHelper, private readonly storageFactory: StorageProviderFactory ) { - this.storageConfig = this.config.storages.avatar; - this.provider = this.storageFactory.create(this.storageConfig); + this.provider = this.storageFactory.create(this.config.storage); } async put(key: string, blob: BlobInputType, metadata?: PutObjectMetadata) { await this.provider.put(key, blob, metadata); - let link = this.storageConfig.publicLinkFactory(key); + let link = this.config.publicPath + key; if (link.startsWith('/')) { link = this.url.link(link); @@ -42,7 +44,7 @@ export class AvatarStorage { } delete(link: string) { - return this.provider.delete(this.storageConfig.keyInPublicLink(link)); + return this.provider.delete(link.split('/').pop() as string); } @OnEvent('user.deleted') @@ -51,4 +53,11 @@ export class AvatarStorage { await this.delete(user.avatarUrl); } } + + @OnEvent('config.changed') + async onConfigChanged(event: Events['config.changed']) { + if (event.updates.storages?.avatar?.storage) { + this.provider = this.storageFactory.create(this.config.storage); + } + } } diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index 980137cdd3..dd5fea880d 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -30,16 +30,20 @@ declare global { @Injectable() export class WorkspaceBlobStorage { private readonly logger = new Logger(WorkspaceBlobStorage.name); - public readonly provider: StorageProvider; + private provider: StorageProvider; + + get config() { + return this.AFFiNEConfig.storages.blob; + } constructor( - private readonly config: Config, + private readonly AFFiNEConfig: Config, private readonly event: EventBus, private readonly storageFactory: StorageProviderFactory, private readonly db: PrismaClient, private readonly url: URLHelper ) { - this.provider = this.storageFactory.create(this.config.storages.blob); + this.provider = this.storageFactory.create(this.config.storage); } async put(workspaceId: string, key: string, blob: Buffer) { @@ -225,4 +229,11 @@ export class WorkspaceBlobStorage { }: Events['workspace.blob.delete']) { await this.delete(workspaceId, key, true); } + + @OnEvent('config.changed') + async onConfigChanged(event: Events['config.changed']) { + if (event.updates.storages?.blob?.storage) { + this.provider = this.storageFactory.create(this.config.storage); + } + } } diff --git a/packages/backend/server/src/core/sync/gateway.ts b/packages/backend/server/src/core/sync/gateway.ts index b99b601ae7..c90e01eb6c 100644 --- a/packages/backend/server/src/core/sync/gateway.ts +++ b/packages/backend/server/src/core/sync/gateway.ts @@ -17,9 +17,7 @@ import { GatewayErrorWrapper, metrics, NotInSpace, - Runtime, SpaceAccessDenied, - VersionRejected, } from '../../base'; import { Models } from '../../models'; import { CurrentUser } from '../auth'; @@ -145,7 +143,6 @@ export class SpaceSyncGateway private connectionCount = 0; constructor( - private readonly runtime: Runtime, private readonly ac: AccessController, private readonly workspace: PgWorkspaceDocStorageAdapter, private readonly userspace: PgUserspaceDocStorageAdapter, @@ -186,30 +183,6 @@ export class SpaceSyncGateway return adapters[spaceType]; } - async assertVersion(client: Socket, version?: string) { - const shouldCheckClientVersion = await this.runtime.fetch( - 'flags/syncClientVersionCheck' - ); - if ( - // @todo(@darkskygit): remove this flag after 0.12 goes stable - shouldCheckClientVersion && - version !== AFFiNE.version - ) { - client.emit('server-version-rejected', { - currentVersion: version, - requiredVersion: AFFiNE.version, - reason: `Client version${ - version ? ` ${version}` : '' - } is outdated, please update to ${AFFiNE.version}`, - }); - - throw new VersionRejected({ - version: version || 'unknown', - serverVersion: AFFiNE.version, - }); - } - } - // v3 @SubscribeMessage('space:join') async onJoinSpace( @@ -218,8 +191,6 @@ export class SpaceSyncGateway @MessageBody() { spaceType, spaceId, clientVersion }: JoinSpaceMessage ): Promise> { - await this.assertVersion(client, clientVersion); - // TODO(@forehalo): remove this after 0.19 goes out of life // simple match 0.19.x if (/^0.19.[\d]$/.test(clientVersion)) { @@ -396,10 +367,8 @@ export class SpaceSyncGateway @ConnectedSocket() client: Socket, @CurrentUser() user: CurrentUser, @MessageBody() - { spaceType, spaceId, docId, clientVersion }: JoinSpaceAwarenessMessage + { spaceType, spaceId, docId }: JoinSpaceAwarenessMessage ) { - await this.assertVersion(client, clientVersion); - await this.selectAdapter(client, spaceType).join( user.id, spaceId, diff --git a/packages/backend/server/src/core/user/controller.ts b/packages/backend/server/src/core/user/controller.ts index 3fec76c61e..46b3575889 100644 --- a/packages/backend/server/src/core/user/controller.ts +++ b/packages/backend/server/src/core/user/controller.ts @@ -12,7 +12,7 @@ export class UserAvatarController { @Get('/:id') async getAvatar(@Res() res: Response, @Param('id') id: string) { - if (this.storage.provider.type !== 'fs') { + if (this.storage.config.storage.provider !== 'fs') { throw new ActionForbidden( 'Only available when avatar storage provider set to fs.' ); diff --git a/packages/backend/server/src/core/user/event.ts b/packages/backend/server/src/core/user/event.ts deleted file mode 100644 index 6dc124f25d..0000000000 --- a/packages/backend/server/src/core/user/event.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; - -import { Config, OnEvent } from '../../base'; - -@Injectable() -export class UserEventsListener { - private readonly logger = new Logger(UserEventsListener.name); - - constructor(private readonly config: Config) {} - - @OnEvent('user.updated') - async onUserUpdated(user: Events['user.updated']) { - const { enabled, customerIo } = this.config.metrics; - if (enabled && customerIo?.token) { - const payload = { - name: user.name, - email: user.email, - created_at: Number(user.createdAt) / 1000, - }; - try { - await fetch(`https://track.customer.io/api/v1/customers/${user.id}`, { - method: 'PUT', - headers: { - Authorization: `Basic ${customerIo.token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); - } catch (e) { - this.logger.error('Failed to publish user update event:', e); - } - } - } - - @OnEvent('user.deleted') - async onUserDeleted(user: Events['user.deleted']) { - const { enabled, customerIo } = this.config.metrics; - if (enabled && customerIo?.token) { - try { - if (user.emailVerifiedAt) { - // suppress email if email is verified - await fetch( - `https://track.customer.io/api/v1/customers/${user.email}/suppress`, - { - method: 'POST', - headers: { - Authorization: `Basic ${customerIo.token}`, - }, - } - ); - } - await fetch(`https://track.customer.io/api/v1/customers/${user.id}`, { - method: 'DELETE', - headers: { Authorization: `Basic ${customerIo.token}` }, - }); - } catch (e) { - this.logger.error('Failed to publish user delete event:', e); - } - } - } -} diff --git a/packages/backend/server/src/core/user/index.ts b/packages/backend/server/src/core/user/index.ts index 14aca45408..8c5d5f83a4 100644 --- a/packages/backend/server/src/core/user/index.ts +++ b/packages/backend/server/src/core/user/index.ts @@ -3,7 +3,6 @@ import { Module } from '@nestjs/common'; import { PermissionModule } from '../permission'; import { StorageModule } from '../storage'; import { UserAvatarController } from './controller'; -import { UserEventsListener } from './event'; import { UserManagementResolver, UserResolver, @@ -12,12 +11,7 @@ import { @Module({ imports: [StorageModule, PermissionModule], - providers: [ - UserResolver, - UserManagementResolver, - UserEventsListener, - UserSettingsResolver, - ], + providers: [UserResolver, UserManagementResolver, UserSettingsResolver], controllers: [UserAvatarController], }) export class UserModule {} diff --git a/packages/backend/server/src/core/version/config.ts b/packages/backend/server/src/core/version/config.ts index 4fb9516b04..62047206e7 100644 --- a/packages/backend/server/src/core/version/config.ts +++ b/packages/backend/server/src/core/version/config.ts @@ -1,4 +1,4 @@ -import { defineRuntimeConfig, ModuleConfig } from '../../base/config'; +import { defineModuleConfig } from '../../base'; export interface VersionConfig { versionControl: { @@ -7,9 +7,9 @@ export interface VersionConfig { }; } -declare module '../../base/config' { - interface AppConfig { - client: ModuleConfig; +declare global { + interface AppConfigSchema { + client: VersionConfig; } } @@ -19,7 +19,7 @@ declare module '../../base/guard' { } } -defineRuntimeConfig('client', { +defineModuleConfig('client', { 'versionControl.enabled': { desc: 'Whether check version of client before accessing the server.', default: false, diff --git a/packages/backend/server/src/core/version/guard.ts b/packages/backend/server/src/core/version/guard.ts index ce83dd99af..ee3ddccb16 100644 --- a/packages/backend/server/src/core/version/guard.ts +++ b/packages/backend/server/src/core/version/guard.ts @@ -6,9 +6,9 @@ import type { import { Injectable } from '@nestjs/common'; import { + Config, getRequestResponseFromContext, GuardProvider, - Runtime, } from '../../base'; import { VersionService } from './service'; @@ -20,14 +20,14 @@ export class VersionGuardProvider name = 'version' as const; constructor( - private readonly runtime: Runtime, + private readonly config: Config, private readonly version: VersionService ) { super(); } async canActivate(context: ExecutionContext) { - if (!(await this.runtime.fetch('client/versionControl.enabled'))) { + if (!this.config.client.versionControl.enabled) { return true; } diff --git a/packages/backend/server/src/core/version/service.ts b/packages/backend/server/src/core/version/service.ts index 9201fde4f6..722094d7ea 100644 --- a/packages/backend/server/src/core/version/service.ts +++ b/packages/backend/server/src/core/version/service.ts @@ -1,18 +1,16 @@ import { Injectable, Logger } from '@nestjs/common'; import semver from 'semver'; -import { Runtime, UnsupportedClientVersion } from '../../base'; +import { Config, UnsupportedClientVersion } from '../../base'; @Injectable() export class VersionService { private readonly logger = new Logger(VersionService.name); - constructor(private readonly runtime: Runtime) {} + constructor(private readonly config: Config) {} async checkVersion(clientVersion?: string) { - const requiredVersion = await this.runtime.fetch( - 'client/versionControl.requiredVersion' - ); + const requiredVersion = this.config.client.versionControl.requiredVersion; const range = await this.getVersionRange(requiredVersion); if (!range) { diff --git a/packages/backend/server/src/env.ts b/packages/backend/server/src/env.ts new file mode 100644 index 0000000000..51b4067a0d --- /dev/null +++ b/packages/backend/server/src/env.ts @@ -0,0 +1,134 @@ +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import pkg from '../package.json' with { type: 'json' }; + +declare global { + namespace globalThis { + // oxlint-disable-next-line no-var + var env: Readonly; + // oxlint-disable-next-line no-var + var readEnv: (key: string, defaultValue: T, availableValues?: T[]) => T; + } +} + +export enum Flavor { + AllInOne = 'allinone', + Graphql = 'graphql', + Sync = 'sync', + Renderer = 'renderer', + Doc = 'doc', + Script = 'script', +} + +export enum Namespace { + Dev = 'dev', + Beta = 'beta', + Production = 'production', +} + +export enum NodeEnv { + Development = 'development', + Test = 'test', + Production = 'production', +} + +export enum DeploymentType { + Affine = 'affine', + Selfhosted = 'selfhosted', +} + +export enum Platform { + GCP = 'gcp', + Unknown = 'unknown', +} + +export type AppEnv = { + NODE_ENV: NodeEnv; + NAMESPACE: Namespace; + DEPLOYMENT_TYPE: DeploymentType; + version: string; +}; + +globalThis.readEnv = function readEnv( + env: string, + defaultValue: T, + availableValues?: T[] +) { + const value = process.env[env]; + if (value === undefined) { + return defaultValue; + } + + if (availableValues && !availableValues.includes(value as any)) { + throw new Error( + `Invalid value "${value}" for environment variable ${env}, expected one of ${JSON.stringify( + availableValues + )}` + ); + } + + return value as T; +}; + +export class Env implements AppEnv { + NODE_ENV = readEnv('NODE_ENV', NodeEnv.Production, Object.values(NodeEnv)); + NAMESPACE = readEnv( + 'AFFINE_ENV', + Namespace.Production, + Object.values(Namespace) + ); + DEPLOYMENT_TYPE = readEnv( + 'DEPLOYMENT_TYPE', + DeploymentType.Affine, + Object.values(DeploymentType) + ); + FLAVOR = readEnv('SERVER_FLAVOR', Flavor.AllInOne, Object.values(Flavor)); + platform = readEnv('DEPLOYMENT_PLATFORM', Platform.Unknown); + version = pkg.version; + projectRoot = resolve(fileURLToPath(import.meta.url), '../../'); + + get selfhosted() { + return this.DEPLOYMENT_TYPE === DeploymentType.Selfhosted; + } + + isFlavor(flavor: Flavor) { + return this.FLAVOR === flavor || this.FLAVOR === Flavor.AllInOne; + } + + get flavors() { + return { + graphql: this.isFlavor(Flavor.Graphql), + sync: this.isFlavor(Flavor.Sync), + renderer: this.isFlavor(Flavor.Renderer), + doc: this.isFlavor(Flavor.Doc), + script: this.isFlavor(Flavor.Script), + }; + } + + get namespaces() { + return { + canary: this.NAMESPACE === Namespace.Dev, + beta: this.NAMESPACE === Namespace.Beta, + production: this.NAMESPACE === Namespace.Production, + }; + } + + get testing() { + return this.NODE_ENV === NodeEnv.Test; + } + + get dev() { + return this.NODE_ENV === NodeEnv.Development; + } + + get prod() { + return this.NODE_ENV === NodeEnv.Production; + } + + get gcp() { + return this.platform === Platform.GCP; + } +} + +globalThis.env = new Env(); diff --git a/packages/backend/server/src/global.d.ts b/packages/backend/server/src/global.d.ts index b0ca1939db..bd734309ba 100644 --- a/packages/backend/server/src/global.d.ts +++ b/packages/backend/server/src/global.d.ts @@ -4,13 +4,26 @@ declare namespace Express { } } +declare type Exact = { + [K in keyof T]: T[K]; +}; + +declare type Leaf = T & { __leaf: true }; +declare type NonLeaf = T extends Leaf ? V : T; + +declare type DeeplyEraseLeaf = T extends Leaf ? V + : + { + [K in keyof T]: DeeplyEraseLeaf + } + declare type PrimitiveType = | string | number | boolean | symbol | null - | undefined; + | undefined declare type UnionToIntersection = ( T extends any ? (x: T) => any : never @@ -33,6 +46,10 @@ declare type DeepPartial = } : T; +declare type DeepReadonly = { + readonly [K in keyof T]: T[K] extends object ? DeepReadonly : T[K]; +}; + declare type AFFiNEModule = | import('@nestjs/common').Type | import('@nestjs/common').DynamicModule; diff --git a/packages/backend/server/src/index.ts b/packages/backend/server/src/index.ts index 8c2dff02a0..a0354c8b0e 100644 --- a/packages/backend/server/src/index.ts +++ b/packages/backend/server/src/index.ts @@ -4,15 +4,17 @@ import './prelude'; import { Logger } from '@nestjs/common'; import { createApp } from './app'; -import { URLHelper } from './base'; +import { Config, URLHelper } from './base'; const app = await createApp(); -const listeningHost = '0.0.0.0'; -await app.listen(AFFiNE.server.port, listeningHost); +const config = app.get(Config); const url = app.get(URLHelper); +const listeningHost = '0.0.0.0'; + +await app.listen(config.server.port, listeningHost); const logger = new Logger('App'); -logger.log(`AFFiNE Server is running in [${AFFiNE.type}] mode`); -logger.log(`Listening on http://${listeningHost}:${AFFiNE.server.port}`); +logger.log(`AFFiNE Server is running in [${env.DEPLOYMENT_TYPE}] mode`); +logger.log(`Listening on http://${listeningHost}:${config.server.port}`); logger.log(`And the public server should be recognized as ${url.home}`); diff --git a/packages/backend/server/src/mails/components/template.tsx b/packages/backend/server/src/mails/components/template.tsx index b4e40d86cd..fc4906f5f5 100644 --- a/packages/backend/server/src/mails/components/template.tsx +++ b/packages/backend/server/src/mails/components/template.tsx @@ -195,7 +195,7 @@ export function Template(props: PropsWithChildren) { ); - if (typeof AFFiNE !== 'undefined' && AFFiNE.node.test) { + if (env.testing) { return content; } diff --git a/packages/backend/server/src/mails/index.tsx b/packages/backend/server/src/mails/index.tsx index f92cceb09b..8f18ec62ed 100644 --- a/packages/backend/server/src/mails/index.tsx +++ b/packages/backend/server/src/mails/index.tsx @@ -41,7 +41,7 @@ type EmailContent = { function render(component: React.ReactElement) { return rawRender(component, { - pretty: AFFiNE.node.test, + pretty: env.testing, }); } @@ -53,7 +53,7 @@ function make>( subject: string | ((props: Props) => string) ): EmailRenderer> { return async props => { - if (!props && AFFiNE.node.test) { + if (!props && env.testing) { // @ts-expect-error test only props = Component.PreviewProps; } diff --git a/packages/backend/server/src/models/base.ts b/packages/backend/server/src/models/base.ts index a182942920..832be3c320 100644 --- a/packages/backend/server/src/models/base.ts +++ b/packages/backend/server/src/models/base.ts @@ -2,7 +2,6 @@ import { Inject, Logger } from '@nestjs/common'; import { TransactionHost } from '@nestjs-cls/transactional'; import type { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-prisma'; -import { Config } from '../base'; import type { Models } from '.'; import { MODELS_SYMBOL } from './provider'; @@ -12,9 +11,6 @@ export class BaseModel { @Inject(MODELS_SYMBOL) protected readonly models!: Models; - @Inject(Config) - protected readonly config!: Config; - @Inject(TransactionHost) private readonly txHost!: TransactionHost; diff --git a/packages/backend/server/src/models/config.ts b/packages/backend/server/src/models/config.ts new file mode 100644 index 0000000000..2847a2945c --- /dev/null +++ b/packages/backend/server/src/models/config.ts @@ -0,0 +1,24 @@ +import { Injectable } from '@nestjs/common'; +import { Transactional } from '@nestjs-cls/transactional'; + +import { BaseModel } from './base'; + +@Injectable() +export class AppConfigModel extends BaseModel { + async load() { + return this.db.appConfig.findMany(); + } + + @Transactional() + async save(user: string, updates: Array<{ key: string; value: any }>) { + return await Promise.allSettled( + updates.map(async update => { + return this.db.appConfig.upsert({ + where: { id: update.key }, + update: { value: update.value, lastUpdatedBy: user }, + create: { id: update.key, value: update.value, lastUpdatedBy: user }, + }); + }) + ); + } +} diff --git a/packages/backend/server/src/models/feature.ts b/packages/backend/server/src/models/feature.ts index 729e5dad72..3f79a700f1 100644 --- a/packages/backend/server/src/models/feature.ts +++ b/packages/backend/server/src/models/feature.ts @@ -133,7 +133,7 @@ export class FeatureModel extends BaseModel { const name = key as FeatureName; const def = FeatureConfigs[name]; // self-hosted instance will use pro plan as free plan - if (name === 'free_plan_v1' && this.config.isSelfhosted) { + if (name === 'free_plan_v1' && env.selfhosted) { await this.upsert( name, FeatureConfigs['pro_plan_v1'].configs, diff --git a/packages/backend/server/src/models/index.ts b/packages/backend/server/src/models/index.ts index f9da66b8bd..e0cbb0754b 100644 --- a/packages/backend/server/src/models/index.ts +++ b/packages/backend/server/src/models/index.ts @@ -7,6 +7,7 @@ import { import { ModuleRef } from '@nestjs/core'; import { ApplyType } from '../base'; +import { AppConfigModel } from './config'; import { CopilotContextModel } from './copilot-context'; import { CopilotJobModel } from './copilot-job'; import { CopilotSessionModel } from './copilot-session'; @@ -44,6 +45,7 @@ const MODELS = { copilotSession: CopilotSessionModel, copilotContext: CopilotContextModel, copilotJob: CopilotJobModel, + appConfig: AppConfigModel, }; type ModelsType = { diff --git a/packages/backend/server/src/models/session.ts b/packages/backend/server/src/models/session.ts index ecef83b501..88c9512e42 100644 --- a/packages/backend/server/src/models/session.ts +++ b/packages/backend/server/src/models/session.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { Prisma, type Session, @@ -6,6 +6,7 @@ import { type UserSession, } from '@prisma/client'; +import { Config } from '../base'; import { BaseModel } from './base'; export type { Session, UserSession }; @@ -13,6 +14,9 @@ export type UserSessionWithUser = UserSession & { user: User }; @Injectable() export class SessionModel extends BaseModel { + @Inject(Config) + private readonly config!: Config; + async createSession() { return await this.db.session.create({ data: {}, diff --git a/packages/backend/server/src/plugins/captcha/config.ts b/packages/backend/server/src/plugins/captcha/config.ts index 3d283109f6..9955fbdcf7 100644 --- a/packages/backend/server/src/plugins/captcha/config.ts +++ b/packages/backend/server/src/plugins/captcha/config.ts @@ -1,18 +1,12 @@ -import { - defineRuntimeConfig, - defineStartupConfig, - ModuleConfig, -} from '../../base/config'; +import { defineModuleConfig } from '../../base'; import { CaptchaConfig } from './types'; -declare module '../config' { - interface PluginsConfig { - captcha: ModuleConfig< - CaptchaConfig, - { - enable: boolean; - } - >; +declare global { + interface AppConfigSchema { + captcha: { + enabled: boolean; + config: ConfigItem; + }; } } @@ -22,18 +16,20 @@ declare module '../../base/guard' { } } -defineStartupConfig('plugins.captcha', { - turnstile: { - secret: '', - }, - challenge: { - bits: 20, - }, -}); - -defineRuntimeConfig('plugins.captcha', { - enable: { +defineModuleConfig('captcha', { + enabled: { desc: 'Check captcha challenge when user authenticating the app.', default: false, }, + config: { + desc: 'The config for the captcha plugin.', + default: { + turnstile: { + secret: '', + }, + challenge: { + bits: 20, + }, + }, + }, }); diff --git a/packages/backend/server/src/plugins/captcha/guard.ts b/packages/backend/server/src/plugins/captcha/guard.ts index 8b8420fca5..ba2afe43bd 100644 --- a/packages/backend/server/src/plugins/captcha/guard.ts +++ b/packages/backend/server/src/plugins/captcha/guard.ts @@ -6,9 +6,9 @@ import type { import { Injectable } from '@nestjs/common'; import { + Config, getRequestResponseFromContext, GuardProvider, - Runtime, } from '../../base'; import { CaptchaService } from './service'; @@ -20,14 +20,14 @@ export class CaptchaGuardProvider name = 'captcha' as const; constructor( - private readonly captcha: CaptchaService, - private readonly runtime: Runtime + private readonly config: Config, + private readonly captcha: CaptchaService ) { super(); } async canActivate(context: ExecutionContext) { - if (!(await this.runtime.fetch('plugins.captcha/enable'))) { + if (!this.config.captcha.enabled) { return true; } diff --git a/packages/backend/server/src/plugins/captcha/index.ts b/packages/backend/server/src/plugins/captcha/index.ts index ac27ad7e2a..17a28d25e6 100644 --- a/packages/backend/server/src/plugins/captcha/index.ts +++ b/packages/backend/server/src/plugins/captcha/index.ts @@ -1,19 +1,17 @@ import './config'; +import { Module } from '@nestjs/common'; + +import { ServerConfigModule } from '../../core'; import { AuthModule } from '../../core/auth'; -import { ServerFeature } from '../../core/config'; -import { Plugin } from '../registry'; import { CaptchaController } from './controller'; import { CaptchaGuardProvider } from './guard'; import { CaptchaService } from './service'; -@Plugin({ - name: 'captcha', - imports: [AuthModule], +@Module({ + imports: [AuthModule, ServerConfigModule], providers: [CaptchaService, CaptchaGuardProvider], controllers: [CaptchaController], - contributesTo: ServerFeature.Captcha, - requires: ['plugins.captcha.turnstile.secret'], }) export class CaptchaModule {} diff --git a/packages/backend/server/src/plugins/captcha/service.ts b/packages/backend/server/src/plugins/captcha/service.ts index 252a13fe9d..c687abdc3c 100644 --- a/packages/backend/server/src/plugins/captcha/service.ts +++ b/packages/backend/server/src/plugins/captcha/service.ts @@ -1,4 +1,3 @@ -import assert from 'node:assert'; import { randomUUID } from 'node:crypto'; import { Injectable, Logger } from '@nestjs/common'; @@ -6,12 +5,10 @@ import type { Request } from 'express'; import { nanoid } from 'nanoid'; import { z } from 'zod'; -import { - CaptchaVerificationFailed, - Config, - verifyChallengeResponse, -} from '../../base'; +import { CaptchaVerificationFailed, Config, OnEvent } from '../../base'; +import { ServerFeature, ServerService } from '../../core'; import { Models, TokenType } from '../../models'; +import { verifyChallengeResponse } from '../../native'; import { CaptchaConfig } from './types'; const validator = z @@ -26,10 +23,22 @@ export class CaptchaService { constructor( private readonly config: Config, - private readonly models: Models + private readonly models: Models, + private readonly server: ServerService ) { - assert(config.plugins.captcha); - this.captcha = config.plugins.captcha; + this.captcha = config.captcha.config; + } + + @OnEvent('config.init') + onConfigInit() { + this.setup(); + } + + @OnEvent('config.changed') + onConfigChanged(event: Events['config.changed']) { + if ('captcha' in event.updates) { + this.setup(); + } } private async verifyCaptchaToken(token: any, ip: string) { @@ -52,7 +61,7 @@ export class CaptchaService { return ( !!outcome.success && // skip hostname check in dev mode - (this.config.node.dev || outcome.hostname === this.config.server.host) + (env.dev || outcome.hostname === this.config.server.host) ); } @@ -119,4 +128,12 @@ export class CaptchaService { } } } + + private setup() { + if (this.config.captcha.enabled) { + this.server.enableFeature(ServerFeature.Captcha); + } else { + this.server.disableFeature(ServerFeature.Captcha); + } + } } diff --git a/packages/backend/server/src/plugins/config.ts b/packages/backend/server/src/plugins/config.ts deleted file mode 100644 index 5f747cccf2..0000000000 --- a/packages/backend/server/src/plugins/config.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { ModuleStartupConfigDescriptions } from '../base/config/types'; - -export interface PluginsConfig {} -export type AvailablePlugins = keyof PluginsConfig; - -declare module '../base/config' { - interface AppConfig { - plugins: PluginsConfig; - } - - interface AppPluginsConfig { - use( - plugin: Plugin, - config?: DeepPartial< - ModuleStartupConfigDescriptions - > - ): void; - plugins: { - /** - * @deprecated use `AFFiNE.use` instead - */ - use( - plugin: Plugin, - config?: DeepPartial< - ModuleStartupConfigDescriptions - > - ): void; - }; - } -} diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts index 6ef3380841..fa3b7dccf1 100644 --- a/packages/backend/server/src/plugins/copilot/config.ts +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -1,30 +1,76 @@ -import type { ClientOptions as OpenAIClientOptions } from 'openai'; - -import { defineStartupConfig, ModuleConfig } from '../../base/config'; -import { StorageConfig } from '../../base/storage/config'; +import { + defineModuleConfig, + StorageJSONSchema, + StorageProviderConfig, +} from '../../base'; import type { FalConfig } from './providers/fal'; -import { GoogleConfig } from './providers/google'; +import { GeminiConfig } from './providers/gemini'; +import { OpenAIConfig } from './providers/openai'; import { PerplexityConfig } from './providers/perplexity'; -export interface CopilotStartupConfigurations { - openai?: OpenAIClientOptions; - fal?: FalConfig; - google?: GoogleConfig; - perplexity?: PerplexityConfig; - test?: never; - unsplashKey?: string; - storage: StorageConfig; -} - -declare module '../config' { - interface PluginsConfig { - copilot: ModuleConfig; +declare global { + interface AppConfigSchema { + copilot: { + enabled: boolean; + unsplash: ConfigItem<{ + key: string; + }>; + storage: ConfigItem; + providers: { + openai: ConfigItem; + fal: ConfigItem; + gemini: ConfigItem; + perplexity: ConfigItem; + }; + }; } } -defineStartupConfig('plugins.copilot', { +defineModuleConfig('copilot', { + enabled: { + desc: 'Whether to enable the copilot plugin.', + default: false, + }, + 'providers.openai': { + desc: 'The config for the openai provider.', + default: { + apiKey: '', + }, + link: 'https://github.com/openai/openai-node', + }, + 'providers.fal': { + desc: 'The config for the fal provider.', + default: { + apiKey: '', + }, + }, + 'providers.gemini': { + desc: 'The config for the gemini provider.', + default: { + apiKey: '', + }, + }, + 'providers.perplexity': { + desc: 'The config for the perplexity provider.', + default: { + apiKey: '', + }, + }, + unsplash: { + desc: 'The config for the unsplash key.', + default: { + key: '', + }, + }, storage: { - provider: 'fs', - bucket: 'copilot', + desc: 'The config for the storage provider.', + default: { + provider: 'fs', + bucket: 'copilot', + config: { + path: '~/.affine/storage', + }, + }, + schema: StorageJSONSchema, }, }); diff --git a/packages/backend/server/src/plugins/copilot/context/job.ts b/packages/backend/server/src/plugins/copilot/context/job.ts index 4b4db101bb..ed74c51eab 100644 --- a/packages/backend/server/src/plugins/copilot/context/job.ts +++ b/packages/backend/server/src/plugins/copilot/context/job.ts @@ -1,4 +1,4 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import OpenAI from 'openai'; import { @@ -37,12 +37,12 @@ declare global { } @Injectable() -export class CopilotContextDocJob implements OnModuleInit { +export class CopilotContextDocJob { private supportEmbedding = false; - private readonly client: EmbeddingClient | undefined; + private client: EmbeddingClient | undefined; constructor( - config: Config, + private readonly config: Config, private readonly doc: DocReader, private readonly event: EventBus, private readonly logger: AFFiNELogger, @@ -51,15 +51,24 @@ export class CopilotContextDocJob implements OnModuleInit { private readonly storage: CopilotStorage ) { this.logger.setContext(CopilotContextDocJob.name); - const configure = config.plugins.copilot.openai; - if (configure) { - this.client = new OpenAIEmbeddingClient(new OpenAI(configure)); - } } - async onModuleInit() { + @OnEvent('config.init') + async onConfigInit() { + await this.setup(); + } + + @OnEvent('config.changed') + async onConfigChanged() { + await this.setup(); + } + + private async setup() { this.supportEmbedding = await this.models.copilotContext.checkEmbeddingAvailable(); + this.client = new OpenAIEmbeddingClient( + new OpenAI(this.config.copilot.providers.openai) + ); } // public this client to allow overriding in tests diff --git a/packages/backend/server/src/plugins/copilot/context/service.ts b/packages/backend/server/src/plugins/copilot/context/service.ts index 05e191f6c2..2e9df05a22 100644 --- a/packages/backend/server/src/plugins/copilot/context/service.ts +++ b/packages/backend/server/src/plugins/copilot/context/service.ts @@ -1,4 +1,4 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; import OpenAI from 'openai'; import { @@ -22,7 +22,7 @@ import { EmbeddingClient } from './types'; const CONTEXT_SESSION_KEY = 'context-session'; @Injectable() -export class CopilotContextService implements OnModuleInit { +export class CopilotContextService implements OnApplicationBootstrap { private supportEmbedding = false; private readonly client: EmbeddingClient | undefined; @@ -31,13 +31,13 @@ export class CopilotContextService implements OnModuleInit { private readonly cache: Cache, private readonly models: Models ) { - const configure = config.plugins.copilot.openai; + const configure = config.copilot.providers.openai; if (configure) { this.client = new OpenAIEmbeddingClient(new OpenAI(configure)); } } - async onModuleInit() { + async onApplicationBootstrap() { const supportEmbedding = await this.models.copilotContext.checkEmbeddingAvailable(); if (supportEmbedding) { diff --git a/packages/backend/server/src/plugins/copilot/controller.ts b/packages/backend/server/src/plugins/copilot/controller.ts index 2d2c1a480c..3cf7c5aebd 100644 --- a/packages/backend/server/src/plugins/copilot/controller.ts +++ b/packages/backend/server/src/plugins/copilot/controller.ts @@ -45,10 +45,14 @@ import { UnsplashIsNotConfigured, } from '../../base'; import { CurrentUser, Public } from '../../core/auth'; -import { CopilotProviderService } from './providers'; +import { + CopilotCapability, + CopilotProviderFactory, + CopilotTextProvider, +} from './providers'; import { ChatSession, ChatSessionService } from './session'; import { CopilotStorage } from './storage'; -import { ChatMessage, CopilotCapability, CopilotTextProvider } from './types'; +import { ChatMessage } from './types'; import { CopilotWorkflowService, GraphExecutorState } from './workflow'; export interface ChatEvent { @@ -72,7 +76,7 @@ export class CopilotController implements BeforeApplicationShutdown { constructor( private readonly config: Config, private readonly chatSession: ChatSessionService, - private readonly provider: CopilotProviderService, + private readonly provider: CopilotProviderFactory, private readonly workflow: CopilotWorkflowService, private readonly storage: CopilotStorage ) {} @@ -121,13 +125,13 @@ export class CopilotController implements BeforeApplicationShutdown { ); let provider = await this.provider.getProviderByCapability( CopilotCapability.TextToText, - model + { model } ); // fallback to image to text if text to text is not available if (!provider && hasAttachment) { provider = await this.provider.getProviderByCapability( CopilotCapability.ImageToText, - model + { model } ); } if (!provider) { @@ -478,7 +482,7 @@ export class CopilotController implements BeforeApplicationShutdown { hasAttachment ? CopilotCapability.ImageToImage : CopilotCapability.TextToImage, - model + { model } ); if (!provider) { throw new NoCopilotProviderAvailable(); @@ -565,8 +569,8 @@ export class CopilotController implements BeforeApplicationShutdown { @Res() res: Response, @Query() params: Record ) { - const { unsplashKey } = this.config.plugins.copilot || {}; - if (!unsplashKey) { + const { key } = this.config.copilot.unsplash; + if (!key) { throw new UnsplashIsNotConfigured(); } @@ -574,7 +578,7 @@ export class CopilotController implements BeforeApplicationShutdown { const response = await fetch( `https://api.unsplash.com/search/photos?${query}`, { - headers: { Authorization: `Client-ID ${unsplashKey}` }, + headers: { Authorization: `Client-ID ${key}` }, signal: this.getSignal(req), } ); diff --git a/packages/backend/server/src/plugins/copilot/index.ts b/packages/backend/server/src/plugins/copilot/index.ts index 3d0b1e69bf..e3f557f32b 100644 --- a/packages/backend/server/src/plugins/copilot/index.ts +++ b/packages/backend/server/src/plugins/copilot/index.ts @@ -1,11 +1,12 @@ import './config'; -import { ServerFeature } from '../../core/config'; +import { Module } from '@nestjs/common'; + +import { ServerConfigModule } from '../../core'; import { DocStorageModule } from '../../core/doc'; import { FeatureModule } from '../../core/features'; import { PermissionModule } from '../../core/permission'; import { QuotaModule } from '../../core/quota'; -import { Plugin } from '../registry'; import { CopilotContextDocJob, CopilotContextResolver, @@ -15,15 +16,7 @@ import { import { CopilotController } from './controller'; import { ChatMessageCache } from './message'; import { PromptService } from './prompt'; -import { - assertProvidersConfigs, - CopilotProviderService, - FalProvider, - OpenAIProvider, - PerplexityProvider, - registerCopilotProvider, -} from './providers'; -import { GoogleProvider } from './providers/google'; +import { CopilotProviderFactory, CopilotProviders } from './providers'; import { CopilotResolver, PromptsManagementResolver, @@ -37,42 +30,39 @@ import { } from './transcript'; import { CopilotWorkflowExecutors, CopilotWorkflowService } from './workflow'; -registerCopilotProvider(FalProvider); -registerCopilotProvider(OpenAIProvider); -registerCopilotProvider(GoogleProvider); -registerCopilotProvider(PerplexityProvider); - -@Plugin({ - name: 'copilot', - imports: [DocStorageModule, FeatureModule, QuotaModule, PermissionModule], +@Module({ + imports: [ + DocStorageModule, + FeatureModule, + QuotaModule, + PermissionModule, + ServerConfigModule, + ], providers: [ + // providers + ...CopilotProviders, + CopilotProviderFactory, + // services ChatSessionService, CopilotResolver, ChatMessageCache, - UserCopilotResolver, PromptService, - CopilotProviderService, CopilotStorage, - PromptsManagementResolver, // workflow CopilotWorkflowService, ...CopilotWorkflowExecutors, // context - CopilotContextRootResolver, CopilotContextResolver, CopilotContextService, CopilotContextDocJob, // transcription CopilotTranscriptionService, CopilotTranscriptionResolver, + // gql resolvers + UserCopilotResolver, + PromptsManagementResolver, + CopilotContextRootResolver, ], controllers: [CopilotController], - contributesTo: ServerFeature.Copilot, - if: config => { - if (config.flavor.graphql || config.flavor.doc) { - return assertProvidersConfigs(config); - } - return false; - }, }) export class CopilotModule {} diff --git a/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts b/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts index b7644f47fc..12cce69bbd 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts @@ -3,12 +3,8 @@ import { Logger } from '@nestjs/common'; import { AiPrompt } from '@prisma/client'; import Mustache from 'mustache'; -import { - getTokenEncoder, - PromptConfig, - PromptMessage, - PromptParams, -} from '../types'; +import { PromptConfig, PromptMessage, PromptParams } from '../providers'; +import { getTokenEncoder } from '../types'; // disable escaping Mustache.escape = (text: string) => text; diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index dd60046574..dc94d908fd 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -1,7 +1,7 @@ import { Logger } from '@nestjs/common'; import { AiPrompt, PrismaClient } from '@prisma/client'; -import { PromptConfig, PromptMessage } from '../types'; +import { PromptConfig, PromptMessage } from '../providers'; type Prompt = Omit< AiPrompt, diff --git a/packages/backend/server/src/plugins/copilot/prompt/service.ts b/packages/backend/server/src/plugins/copilot/prompt/service.ts index fadf6b220c..52d21f6f30 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/service.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/service.ts @@ -1,4 +1,4 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable, OnApplicationBootstrap } from '@nestjs/common'; import { PrismaClient } from '@prisma/client'; import { @@ -6,17 +6,17 @@ import { PromptConfigSchema, PromptMessage, PromptMessageSchema, -} from '../types'; +} from '../providers'; import { ChatPrompt } from './chat-prompt'; import { refreshPrompts } from './prompts'; @Injectable() -export class PromptService implements OnModuleInit { +export class PromptService implements OnApplicationBootstrap { private readonly cache = new Map(); constructor(private readonly db: PrismaClient) {} - async onModuleInit() { + async onApplicationBootstrap() { this.cache.clear(); await refreshPrompts(this.db); } diff --git a/packages/backend/server/src/plugins/copilot/providers/factory.ts b/packages/backend/server/src/plugins/copilot/providers/factory.ts new file mode 100644 index 0000000000..7a938fc27a --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/providers/factory.ts @@ -0,0 +1,107 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { ServerFeature, ServerService } from '../../../core'; +import type { FalProvider } from './fal'; +import type { GeminiProvider } from './gemini'; +import type { OpenAIProvider } from './openai'; +import type { PerplexityProvider } from './perplexity'; +import type { CopilotProvider } from './provider'; +import { + CapabilityToCopilotProvider, + CopilotCapability, + CopilotProviderType, +} from './types'; + +type TypedProvider = { + [CopilotProviderType.Gemini]: GeminiProvider; + [CopilotProviderType.OpenAI]: OpenAIProvider; + [CopilotProviderType.Perplexity]: PerplexityProvider; + [CopilotProviderType.FAL]: FalProvider; +}; + +@Injectable() +export class CopilotProviderFactory { + constructor(private readonly server: ServerService) {} + + private readonly logger = new Logger(CopilotProviderFactory.name); + + readonly #providers = new Map(); + + getProvider

(provider: P): TypedProvider[P] { + return this.#providers.get(provider) as TypedProvider[P]; + } + + async getProviderByCapability( + capability: C, + filter: { + model?: string; + prefer?: CopilotProviderType; + } = {} + ): Promise { + this.logger.debug( + `Resolving copilot provider for capability: ${capability}` + ); + let candidate: CopilotProvider | null = null; + for (const [type, provider] of this.#providers.entries()) { + // we firstly match by capability + if (provider.capabilities.includes(capability)) { + // use the first match if no filter provided + if (!filter.model && !filter.prefer) { + candidate = provider; + this.logger.debug(`Copilot provider candidate found: ${type}`); + break; + } + + if ( + (!filter.model || (await provider.isModelAvailable(filter.model))) && + (!filter.prefer || filter.prefer === type) + ) { + candidate = provider; + this.logger.debug(`Copilot provider candidate found: ${type}`); + break; + } + } + } + + return candidate as CapabilityToCopilotProvider[C] | null; + } + + async getProviderByModel( + model: string, + filter: { + prefer?: CopilotProviderType; + } = {} + ): Promise { + this.logger.debug(`Resolving copilot provider for model: ${model}`); + + let candidate: CopilotProvider | null = null; + for (const [type, provider] of this.#providers.entries()) { + // we firstly match by model + if (await provider.isModelAvailable(model)) { + candidate = provider; + this.logger.debug(`Copilot provider candidate found: ${type}`); + + // then we match by prefer filter + if (!filter.prefer || filter.prefer === type) { + candidate = provider; + } + } + } + + return candidate as CapabilityToCopilotProvider[C] | null; + } + + register(provider: CopilotProvider) { + this.#providers.set(provider.type, provider); + this.logger.log(`Copilot provider [${provider.type}] registered.`); + this.server.enableFeature(ServerFeature.Copilot); + } + + unregister(provider: CopilotProvider) { + this.#providers.delete(provider.type); + this.logger.log(`Copilot provider [${provider.type}] unregistered.`); + if (this.#providers.size === 0) { + this.server.disableFeature(ServerFeature.Copilot); + } + } +} diff --git a/packages/backend/server/src/plugins/copilot/providers/fal.ts b/packages/backend/server/src/plugins/copilot/providers/fal.ts index 4e9f32257c..238c3f9a19 100644 --- a/packages/backend/server/src/plugins/copilot/providers/fal.ts +++ b/packages/backend/server/src/plugins/copilot/providers/fal.ts @@ -1,9 +1,8 @@ -import assert from 'node:assert'; - import { config as falConfig, stream as falStream, } from '@fal-ai/serverless-client'; +import { Injectable } from '@nestjs/common'; import { z, ZodType } from 'zod'; import { @@ -12,6 +11,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; +import { CopilotProvider } from './provider'; import { CopilotCapability, CopilotChatOptions, @@ -20,7 +20,7 @@ import { CopilotProviderType, CopilotTextToImageProvider, PromptMessage, -} from '../types'; +} from './types'; export type FalConfig = { apiKey: string; @@ -71,17 +71,19 @@ type FalPrompt = { }[]; }; +@Injectable() export class FalProvider + extends CopilotProvider implements CopilotTextToImageProvider, CopilotImageToImageProvider { - static readonly type = CopilotProviderType.FAL; - static readonly capabilities = [ + override type = CopilotProviderType.FAL; + override readonly capabilities = [ CopilotCapability.TextToImage, CopilotCapability.ImageToImage, CopilotCapability.ImageToText, ]; - readonly availableModels = [ + override readonly models = [ // text to image 'fast-turbo-diffusion', // image to image @@ -96,27 +98,15 @@ export class FalProvider 'llava-next', ]; - constructor(private readonly config: FalConfig) { - assert(FalProvider.assetsConfig(config)); + override configured(): boolean { + return !!this.config.apiKey; + } + + protected override setup() { + super.setup(); falConfig({ credentials: this.config.apiKey }); } - static assetsConfig(config: FalConfig) { - return !!config.apiKey; - } - - get type(): CopilotProviderType { - return FalProvider.type; - } - - getCapabilities(): CopilotCapability[] { - return FalProvider.capabilities; - } - - async isModelAvailable(model: string): Promise { - return this.availableModels.includes(model); - } - private extractArray(value: T | T[] | undefined): T[] { if (Array.isArray(value)) return value; return value ? [value] : []; @@ -211,7 +201,7 @@ export class FalProvider model: string = 'llava-next', options: CopilotChatOptions = {} ): Promise { - if (!this.availableModels.includes(model)) { + if (!(await this.isModelAvailable(model))) { throw new CopilotPromptInvalid(`Invalid model: ${model}`); } @@ -269,7 +259,7 @@ export class FalProvider private async buildResponse( messages: PromptMessage[], - model: string = this.availableModels[0], + model: string = this.models[0], options: CopilotImageOptions = {} ) { // by default, image prompt assumes there is only one message @@ -300,10 +290,10 @@ export class FalProvider // ====== image to image ====== async generateImages( messages: PromptMessage[], - model: string = this.availableModels[0], + model: string = this.models[0], options: CopilotImageOptions = {} ): Promise> { - if (!this.availableModels.includes(model)) { + if (!(await this.isModelAvailable(model))) { throw new CopilotPromptInvalid(`Invalid model: ${model}`); } @@ -333,7 +323,7 @@ export class FalProvider async *generateImagesStream( messages: PromptMessage[], - model: string = this.availableModels[0], + model: string = this.models[0], options: CopilotImageOptions = {} ): AsyncIterable { try { diff --git a/packages/backend/server/src/plugins/copilot/providers/google.ts b/packages/backend/server/src/plugins/copilot/providers/gemini.ts similarity index 87% rename from packages/backend/server/src/plugins/copilot/providers/google.ts rename to packages/backend/server/src/plugins/copilot/providers/gemini.ts index ee8182519e..f9f966e69e 100644 --- a/packages/backend/server/src/plugins/copilot/providers/google.ts +++ b/packages/backend/server/src/plugins/copilot/providers/gemini.ts @@ -2,7 +2,6 @@ import { createGoogleGenerativeAI, type GoogleGenerativeAIProvider, } from '@ai-sdk/google'; -import { Logger } from '@nestjs/common'; import { AISDKError, type CoreAssistantMessage, @@ -19,6 +18,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; +import { CopilotProvider } from './provider'; import { ChatMessageRole, CopilotCapability, @@ -26,7 +26,7 @@ import { CopilotProviderType, CopilotTextToTextProvider, PromptMessage, -} from '../types'; +} from './types'; export const DEFAULT_DIMENSIONS = 256; @@ -49,45 +49,38 @@ const FORMAT_INFER_MAP: Record = { flv: 'video/flv', }; -export type GoogleConfig = { +export type GeminiConfig = { apiKey: string; baseUrl?: string; }; type ChatMessage = CoreUserMessage | CoreAssistantMessage; -export class GoogleProvider implements CopilotTextToTextProvider { - static readonly type = CopilotProviderType.Google; - static readonly capabilities = [CopilotCapability.TextToText]; - - readonly availableModels = [ +export class GeminiProvider + extends CopilotProvider + implements CopilotTextToTextProvider +{ + override readonly type = CopilotProviderType.Gemini; + override readonly capabilities = [CopilotCapability.TextToText]; + override readonly models = [ // text to text 'gemini-2.0-flash-001', // embeddings 'text-embedding-004', ]; - private readonly logger = new Logger(GoogleProvider.name); - private readonly instance: GoogleGenerativeAIProvider; + #instance!: GoogleGenerativeAIProvider; - constructor(config: GoogleConfig) { - this.instance = createGoogleGenerativeAI(config); + override configured(): boolean { + return !!this.config.apiKey; } - static assetsConfig(config: GoogleConfig) { - return !!config?.apiKey; - } - - get type(): CopilotProviderType { - return GoogleProvider.type; - } - - getCapabilities(): CopilotCapability[] { - return GoogleProvider.capabilities; - } - - async isModelAvailable(model: string): Promise { - return this.availableModels.includes(model); + protected override setup() { + super.setup(); + this.#instance = createGoogleGenerativeAI({ + apiKey: this.config.apiKey, + baseURL: this.config.baseUrl, + }); } private inferMimeType(url: string) { @@ -239,7 +232,7 @@ export class GoogleProvider implements CopilotTextToTextProvider { const [system, msgs] = await this.chatToGPTMessage(messages); const { text } = await generateText({ - model: this.instance(model, { + model: this.#instance(model, { audioTimestamp: Boolean(options.audioTimestamp), structuredOutputs: Boolean(options.jsonMode), }), @@ -268,7 +261,7 @@ export class GoogleProvider implements CopilotTextToTextProvider { const [system, msgs] = await this.chatToGPTMessage(messages); const { textStream } = streamText({ - model: this.instance(model), + model: this.#instance(model), system, messages: msgs, abortSignal: options.signal, diff --git a/packages/backend/server/src/plugins/copilot/providers/index.ts b/packages/backend/server/src/plugins/copilot/providers/index.ts index d4f663ef6f..0168e22503 100644 --- a/packages/backend/server/src/plugins/copilot/providers/index.ts +++ b/packages/backend/server/src/plugins/copilot/providers/index.ts @@ -1,203 +1,18 @@ -import assert from 'node:assert'; +import { FalProvider } from './fal'; +import { GeminiProvider } from './gemini'; +import { OpenAIProvider } from './openai'; +import { PerplexityProvider } from './perplexity'; -import { Injectable, Logger } from '@nestjs/common'; - -import { AFFiNEConfig, Config } from '../../../base'; -import { CopilotStartupConfigurations } from '../config'; -import { - CapabilityToCopilotProvider, - CopilotCapability, - CopilotProvider, - CopilotProviderType, -} from '../types'; - -type CopilotProviderConfig = - CopilotStartupConfigurations[keyof CopilotStartupConfigurations]; - -interface CopilotProviderDefinition { - // constructor signature - new (config: C): CopilotProvider; - // type of the provider - readonly type: CopilotProviderType; - // capabilities of the provider, like text to text, text to image, etc. - readonly capabilities: CopilotCapability[]; - // asserts that the config is valid for this provider - assetsConfig(config: C): boolean; -} - -// registered provider factory -const COPILOT_PROVIDER = new Map< - CopilotProviderType, - (config: Config, logger: Logger) => CopilotProvider ->(); - -// map of capabilities to providers -const PROVIDER_CAPABILITY_MAP = new Map< - CopilotCapability, - CopilotProviderType[] ->(); - -// config assertions for providers -const ASSERT_CONFIG = new Map< - CopilotProviderType, - (config: AFFiNEConfig) => void ->(); - -export function registerCopilotProvider< - C extends CopilotProviderConfig = CopilotProviderConfig, ->(provider: CopilotProviderDefinition) { - const type = provider.type; - - const factory = (config: Config, logger: Logger) => { - const providerConfig = config.plugins.copilot?.[type]; - if (!provider.assetsConfig(providerConfig as C)) { - throw new Error( - `Invalid configuration for copilot provider ${type}: ${JSON.stringify(providerConfig)}` - ); - } - const instance = new provider(providerConfig as C); - logger.debug( - `Copilot provider ${type} registered, capabilities: ${provider.capabilities.join(', ')}` - ); - - return instance; - }; - // register the provider - COPILOT_PROVIDER.set(type, factory); - // register the provider capabilities - for (const capability of provider.capabilities) { - const providers = PROVIDER_CAPABILITY_MAP.get(capability) || []; - if (!providers.includes(type)) { - providers.push(type); - } - PROVIDER_CAPABILITY_MAP.set(capability, providers); - } - // register the provider config assertion - ASSERT_CONFIG.set(type, (config: AFFiNEConfig) => { - assert(config.plugins.copilot); - const providerConfig = config.plugins.copilot[type]; - if (!providerConfig) return false; - return provider.assetsConfig(providerConfig as C); - }); -} - -export function unregisterCopilotProvider(type: CopilotProviderType) { - COPILOT_PROVIDER.delete(type); - ASSERT_CONFIG.delete(type); - for (const providers of PROVIDER_CAPABILITY_MAP.values()) { - const index = providers.indexOf(type); - if (index !== -1) { - providers.splice(index, 1); - } - } -} - -/// Asserts that the config is valid for any registered providers -export function assertProvidersConfigs(config: AFFiNEConfig) { - return Array.from(ASSERT_CONFIG.values()).some(assertConfig => - assertConfig(config) - ); -} - -@Injectable() -export class CopilotProviderService { - private readonly logger = new Logger(CopilotProviderService.name); - constructor(private readonly config: Config) {} - - private readonly cachedProviders = new Map< - CopilotProviderType, - CopilotProvider - >(); - - private create(provider: CopilotProviderType): CopilotProvider { - assert(this.config.plugins.copilot); - const providerFactory = COPILOT_PROVIDER.get(provider); - - if (!providerFactory) { - throw new Error(`Unknown copilot provider type: ${provider}`); - } - - return providerFactory(this.config, this.logger); - } - - getProvider(provider: CopilotProviderType): CopilotProvider | null { - try { - if (!this.cachedProviders.has(provider)) { - this.cachedProviders.set(provider, this.create(provider)); - } - return this.cachedProviders.get(provider) as CopilotProvider; - } catch { - // skip if the provider is not available - return null; - } - } - - async getProviderByCapability( - capability: C, - model?: string, - prefer?: CopilotProviderType - ): Promise { - const providers = PROVIDER_CAPABILITY_MAP.get(capability); - if (Array.isArray(providers) && providers.length) { - let selectedProvider: CopilotProviderType | undefined = prefer; - let currentIndex = -1; - - if (!selectedProvider) { - currentIndex = 0; - selectedProvider = providers[currentIndex]; - } - - while (selectedProvider) { - // find first provider that supports the capability and model - if (providers.includes(selectedProvider)) { - const provider = this.getProvider(selectedProvider); - - if (provider && provider.getCapabilities().includes(capability)) { - if (model) { - if (await provider.isModelAvailable(model)) { - return provider as CapabilityToCopilotProvider[C]; - } - } else { - return provider as CapabilityToCopilotProvider[C]; - } - } - } - currentIndex += 1; - selectedProvider = providers[currentIndex]; - } - } - return null; - } - - async getProviderByModel( - model: string, - prefer?: CopilotProviderType - ): Promise { - const providers = Array.from(COPILOT_PROVIDER.keys()); - if (providers.length) { - let selectedProvider: CopilotProviderType | undefined = prefer; - let currentIndex = -1; - - if (!selectedProvider) { - currentIndex = 0; - selectedProvider = providers[currentIndex]; - } - - while (selectedProvider) { - const provider = this.getProvider(selectedProvider); - - if (provider && (await provider.isModelAvailable(model))) { - return provider as CapabilityToCopilotProvider[C]; - } - - currentIndex += 1; - selectedProvider = providers[currentIndex]; - } - } - return null; - } -} +export const CopilotProviders = [ + OpenAIProvider, + FalProvider, + GeminiProvider, + PerplexityProvider, +]; +export { CopilotProviderFactory } from './factory'; export { FalProvider } from './fal'; +export { GeminiProvider } from './gemini'; export { OpenAIProvider } from './openai'; export { PerplexityProvider } from './perplexity'; +export * from './types'; diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 87eefa23ff..db4da5cf84 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -1,4 +1,3 @@ -import { Logger } from '@nestjs/common'; import { APIError, BadRequestError, ClientOptions, OpenAI } from 'openai'; import { @@ -7,6 +6,7 @@ import { metrics, UserFriendlyError, } from '../../../base'; +import { CopilotProvider } from './provider'; import { ChatMessageRole, CopilotCapability, @@ -19,28 +19,31 @@ import { CopilotTextToImageProvider, CopilotTextToTextProvider, PromptMessage, -} from '../types'; +} from './types'; export const DEFAULT_DIMENSIONS = 256; const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/; +export type OpenAIConfig = ClientOptions; + export class OpenAIProvider + extends CopilotProvider implements CopilotTextToTextProvider, CopilotTextToEmbeddingProvider, CopilotTextToImageProvider, CopilotImageToTextProvider { - static readonly type = CopilotProviderType.OpenAI; - static readonly capabilities = [ + readonly type = CopilotProviderType.OpenAI; + readonly capabilities = [ CopilotCapability.TextToText, CopilotCapability.TextToEmbedding, CopilotCapability.TextToImage, CopilotCapability.ImageToText, ]; - readonly availableModels = [ + readonly models = [ // text to text 'gpt-4o', 'gpt-4o-2024-08-06', @@ -59,41 +62,32 @@ export class OpenAIProvider 'dall-e-3', ]; - private readonly logger = new Logger(OpenAIProvider.type); - private readonly instance: OpenAI; + #existsModels: string[] = []; + #instance!: OpenAI; - private existsModels: string[] | undefined; - - constructor(config: ClientOptions) { - this.instance = new OpenAI(config); + override configured(): boolean { + return !!this.config.apiKey; } - static assetsConfig(config: ClientOptions) { - return !!config?.apiKey; + protected override setup() { + super.setup(); + this.#instance = new OpenAI(this.config); } - get type(): CopilotProviderType { - return OpenAIProvider.type; - } - - getCapabilities(): CopilotCapability[] { - return OpenAIProvider.capabilities; - } - - async isModelAvailable(model: string): Promise { - const knownModels = this.availableModels.includes(model); + override async isModelAvailable(model: string): Promise { + const knownModels = this.models.includes(model); if (knownModels) return true; - if (!this.existsModels) { + if (!this.#existsModels) { try { - this.existsModels = await this.instance.models + this.#existsModels = await this.#instance.models .list() .then(({ data }) => data.map(m => m.id)); } catch (e: any) { this.logger.error('Failed to fetch online model list', e.stack); } } - return !!this.existsModels?.includes(model); + return !!this.#existsModels?.includes(model); } protected chatToGPTMessage( @@ -226,7 +220,7 @@ export class OpenAIProvider try { metrics.ai.counter('chat_text_calls').add(1, { model }); - const result = await this.instance.chat.completions.create( + const result = await this.#instance.chat.completions.create( { messages: this.chatToGPTMessage(messages), model: model, @@ -257,7 +251,7 @@ export class OpenAIProvider try { metrics.ai.counter('chat_text_stream_calls').add(1, { model }); - const result = await this.instance.chat.completions.create( + const result = await this.#instance.chat.completions.create( { stream: true, messages: this.chatToGPTMessage(messages), @@ -307,7 +301,7 @@ export class OpenAIProvider try { metrics.ai.counter('generate_embedding_calls').add(1, { model }); - const result = await this.instance.embeddings.create({ + const result = await this.#instance.embeddings.create({ model: model, input: messages, dimensions: options.dimensions || DEFAULT_DIMENSIONS, @@ -333,7 +327,7 @@ export class OpenAIProvider try { metrics.ai.counter('generate_images_calls').add(1, { model }); - const result = await this.instance.images.generate( + const result = await this.#instance.images.generate( { prompt, model, diff --git a/packages/backend/server/src/plugins/copilot/providers/perplexity.ts b/packages/backend/server/src/plugins/copilot/providers/perplexity.ts index d355644743..c01659c660 100644 --- a/packages/backend/server/src/plugins/copilot/providers/perplexity.ts +++ b/packages/backend/server/src/plugins/copilot/providers/perplexity.ts @@ -1,5 +1,3 @@ -import assert from 'node:assert'; - import { EventSourceParserStream } from 'eventsource-parser/stream'; import { z } from 'zod'; @@ -8,13 +6,14 @@ import { CopilotProviderSideError, metrics, } from '../../../base'; +import { CopilotProvider } from './provider'; import { CopilotCapability, CopilotChatOptions, CopilotProviderType, CopilotTextToTextProvider, PromptMessage, -} from '../types'; +} from './types'; export type PerplexityConfig = { apiKey: string; @@ -164,36 +163,21 @@ export class CitationParser { } } -export class PerplexityProvider implements CopilotTextToTextProvider { - static readonly type = CopilotProviderType.Perplexity; - - static readonly capabilities = [CopilotCapability.TextToText]; - - static assetsConfig(config: PerplexityConfig) { - return !!config.apiKey; - } - - constructor(private readonly config: PerplexityConfig) { - assert(PerplexityProvider.assetsConfig(config)); - } - - readonly availableModels = [ +export class PerplexityProvider + extends CopilotProvider + implements CopilotTextToTextProvider +{ + readonly type = CopilotProviderType.Perplexity; + readonly capabilities = [CopilotCapability.TextToText]; + readonly models = [ 'sonar', 'sonar-pro', 'sonar-reasoning', 'sonar-reasoning-pro', ]; - get type(): CopilotProviderType { - return PerplexityProvider.type; - } - - getCapabilities(): CopilotCapability[] { - return PerplexityProvider.capabilities; - } - - async isModelAvailable(model: string): Promise { - return this.availableModels.includes(model); + override configured(): boolean { + return !!this.config.apiKey; } async generateText( diff --git a/packages/backend/server/src/plugins/copilot/providers/provider.ts b/packages/backend/server/src/plugins/copilot/providers/provider.ts new file mode 100644 index 0000000000..dd8f9111e4 --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/providers/provider.ts @@ -0,0 +1,45 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; + +import { Config, OnEvent } from '../../../base'; +import { CopilotProviderFactory } from './factory'; +import { CopilotCapability, CopilotProviderType } from './types'; + +@Injectable() +export abstract class CopilotProvider { + protected readonly logger = new Logger(this.constructor.name); + abstract readonly type: CopilotProviderType; + abstract readonly capabilities: CopilotCapability[]; + abstract readonly models: string[]; + abstract configured(): boolean; + + @Inject() protected readonly AFFiNEConfig!: Config; + @Inject() protected readonly factory!: CopilotProviderFactory; + + get config(): C { + return this.AFFiNEConfig.copilot.providers[this.type] as C; + } + + isModelAvailable(model: string): Promise | boolean { + return this.models.includes(model); + } + + @OnEvent('config.init') + async onConfigInit() { + this.setup(); + } + + @OnEvent('config.changed') + async onConfigChanged(event: Events['config.changed']) { + if ('copilot' in event.updates) { + this.setup(); + } + } + + protected setup() { + if (this.configured()) { + this.factory.register(this); + } else { + this.factory.unregister(this); + } + } +} diff --git a/packages/backend/server/src/plugins/copilot/providers/types.ts b/packages/backend/server/src/plugins/copilot/providers/types.ts new file mode 100644 index 0000000000..786e1413f4 --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/providers/types.ts @@ -0,0 +1,170 @@ +import { AiPromptRole } from '@prisma/client'; +import { z } from 'zod'; + +import { type CopilotProvider } from './provider'; + +export enum CopilotProviderType { + FAL = 'fal', + Gemini = 'gemini', + OpenAI = 'openai', + Perplexity = 'perplexity', +} + +export enum CopilotCapability { + TextToText = 'text-to-text', + TextToEmbedding = 'text-to-embedding', + TextToImage = 'text-to-image', + ImageToImage = 'image-to-image', + ImageToText = 'image-to-text', +} + +export const PromptConfigStrictSchema = z.object({ + // openai + jsonMode: z.boolean().nullable().optional(), + frequencyPenalty: z.number().nullable().optional(), + presencePenalty: z.number().nullable().optional(), + temperature: z.number().nullable().optional(), + topP: z.number().nullable().optional(), + maxTokens: z.number().nullable().optional(), + // fal + modelName: z.string().nullable().optional(), + loras: z + .array( + z.object({ path: z.string(), scale: z.number().nullable().optional() }) + ) + .nullable() + .optional(), + // google + audioTimestamp: z.boolean().nullable().optional(), +}); + +export const PromptConfigSchema = + PromptConfigStrictSchema.nullable().optional(); + +export type PromptConfig = z.infer; + +export const ChatMessageRole = Object.values(AiPromptRole) as [ + 'system', + 'assistant', + 'user', +]; + +export const PureMessageSchema = z.object({ + content: z.string(), + attachments: z.array(z.string()).optional().nullable(), + params: z.record(z.any()).optional().nullable(), +}); + +export const PromptMessageSchema = PureMessageSchema.extend({ + role: z.enum(ChatMessageRole), +}).strict(); +export type PromptMessage = z.infer; +export type PromptParams = NonNullable; + +const CopilotProviderOptionsSchema = z.object({ + signal: z.instanceof(AbortSignal).optional(), + user: z.string().optional(), +}); + +const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge( + PromptConfigStrictSchema +).optional(); + +export type CopilotChatOptions = z.infer; + +const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({ + dimensions: z.number(), +}).optional(); + +export type CopilotEmbeddingOptions = z.infer< + typeof CopilotEmbeddingOptionsSchema +>; + +const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge( + PromptConfigStrictSchema +) + .extend({ + seed: z.number().optional(), + }) + .optional(); + +export type CopilotImageOptions = z.infer; + +export interface CopilotTextToTextProvider extends CopilotProvider { + generateText( + messages: PromptMessage[], + model?: string, + options?: CopilotChatOptions + ): Promise; + generateTextStream( + messages: PromptMessage[], + model?: string, + options?: CopilotChatOptions + ): AsyncIterable; +} + +export interface CopilotTextToEmbeddingProvider extends CopilotProvider { + generateEmbedding( + messages: string[] | string, + model: string, + options?: CopilotEmbeddingOptions + ): Promise; +} + +export interface CopilotTextToImageProvider extends CopilotProvider { + generateImages( + messages: PromptMessage[], + model: string, + options?: CopilotImageOptions + ): Promise>; + generateImagesStream( + messages: PromptMessage[], + model?: string, + options?: CopilotImageOptions + ): AsyncIterable; +} + +export interface CopilotImageToTextProvider extends CopilotProvider { + generateText( + messages: PromptMessage[], + model: string, + options?: CopilotChatOptions + ): Promise; + generateTextStream( + messages: PromptMessage[], + model: string, + options?: CopilotChatOptions + ): AsyncIterable; +} + +export interface CopilotImageToImageProvider extends CopilotProvider { + generateImages( + messages: PromptMessage[], + model: string, + options?: CopilotImageOptions + ): Promise>; + generateImagesStream( + messages: PromptMessage[], + model?: string, + options?: CopilotImageOptions + ): AsyncIterable; +} + +export type CapabilityToCopilotProvider = { + [CopilotCapability.TextToText]: CopilotTextToTextProvider; + [CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider; + [CopilotCapability.TextToImage]: CopilotTextToImageProvider; + [CopilotCapability.ImageToText]: CopilotImageToTextProvider; + [CopilotCapability.ImageToImage]: CopilotImageToImageProvider; +}; + +export type CopilotTextProvider = + | CopilotTextToTextProvider + | CopilotImageToTextProvider; +export type CopilotImageProvider = + | CopilotTextToImageProvider + | CopilotImageToImageProvider; +export type CopilotAllProvider = + | CopilotTextProvider + | CopilotImageProvider + | CopilotTextToEmbeddingProvider; diff --git a/packages/backend/server/src/plugins/copilot/session.ts b/packages/backend/server/src/plugins/copilot/session.ts index bf2f90f1d0..e0a1f2007c 100644 --- a/packages/backend/server/src/plugins/copilot/session.ts +++ b/packages/backend/server/src/plugins/copilot/session.ts @@ -17,6 +17,7 @@ import { QuotaService } from '../../core/quota'; import { Models } from '../../models'; import { ChatMessageCache } from './message'; import { PromptService } from './prompt'; +import { PromptMessage, PromptParams } from './providers'; import { AvailableModel, ChatHistory, @@ -28,8 +29,6 @@ import { ChatSessionState, getTokenEncoder, ListHistoriesOptions, - PromptMessage, - PromptParams, SubmittedMessage, } from './types'; @@ -533,15 +532,17 @@ export class ChatSessionService { const ret = ChatMessageSchema.array().safeParse(messages); if (ret.success) { // render system prompt - const preload = options?.withPrompt - ? prompt - .finish(ret.data[0]?.params || {}, id) - .filter(({ role }) => role !== 'system') - : []; + const preload = ( + options?.withPrompt + ? prompt + .finish(ret.data[0]?.params || {}, id) + .filter(({ role }) => role !== 'system') + : [] + ) as ChatMessage[]; // `createdAt` is required for history sorting in frontend // let's fake the creating time of prompt messages - (preload as ChatMessage[]).forEach((msg, i) => { + preload.forEach((msg, i) => { msg.createdAt = new Date( createdAt.getTime() - preload.length - i - 1 ); diff --git a/packages/backend/server/src/plugins/copilot/storage.ts b/packages/backend/server/src/plugins/copilot/storage.ts index 0e6830dedc..12b2fb613a 100644 --- a/packages/backend/server/src/plugins/copilot/storage.ts +++ b/packages/backend/server/src/plugins/copilot/storage.ts @@ -25,9 +25,7 @@ export class CopilotStorage { private readonly storageFactory: StorageProviderFactory, private readonly quota: QuotaService ) { - this.provider = this.storageFactory.create( - this.config.plugins.copilot.storage - ); + this.provider = this.storageFactory.create(this.config.copilot.storage); } @CallMetric('ai', 'blob_put') @@ -39,7 +37,7 @@ export class CopilotStorage { ) { const name = `${userId}/${workspaceId}/${key}`; await this.provider.put(name, blob); - if (this.config.node.dev || this.config.node.test) { + if (!env.prod) { // return image base64url for dev environment return `data:image/png;base64,${blob.toString('base64')}`; } diff --git a/packages/backend/server/src/plugins/copilot/transcript/service.ts b/packages/backend/server/src/plugins/copilot/transcript/service.ts index 070c37f9ae..c5529f326c 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/service.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts @@ -11,13 +11,13 @@ import { } from '../../../base'; import { Models } from '../../../models'; import { PromptService } from '../prompt'; -import { CopilotProviderService } from '../providers'; -import { CopilotStorage } from '../storage'; import { CopilotCapability, + CopilotProviderFactory, CopilotTextProvider, PromptMessage, -} from '../types'; +} from '../providers'; +import { CopilotStorage } from '../storage'; import { TranscriptionPayload, TranscriptionSchema, @@ -38,7 +38,7 @@ export class CopilotTranscriptionService { private readonly job: JobQueue, private readonly storage: CopilotStorage, private readonly prompt: PromptService, - private readonly provider: CopilotProviderService + private readonly providerFactory: CopilotProviderFactory ) {} async submitTranscriptionJob( @@ -123,9 +123,9 @@ export class CopilotTranscriptionService { } private async getProvider(model: string): Promise { - let provider = await this.provider.getProviderByCapability( + let provider = await this.providerFactory.getProviderByCapability( CopilotCapability.TextToText, - model + { model } ); if (!provider) { diff --git a/packages/backend/server/src/plugins/copilot/types.ts b/packages/backend/server/src/plugins/copilot/types.ts index e4a89dc2e8..fcde77d47a 100644 --- a/packages/backend/server/src/plugins/copilot/types.ts +++ b/packages/backend/server/src/plugins/copilot/types.ts @@ -1,9 +1,9 @@ import { type Tokenizer } from '@affine/server-native'; -import { AiPromptRole } from '@prisma/client'; import { z } from 'zod'; import { fromModelName } from '../../native'; import type { ChatPrompt } from './prompt'; +import { PromptMessageSchema, PureMessageSchema } from './providers'; export enum AvailableModels { // text to text @@ -41,77 +41,30 @@ export function getTokenEncoder(model?: string | null): Tokenizer | null { // ======== ChatMessage ======== -export const ChatMessageRole = Object.values(AiPromptRole) as [ - 'system', - 'assistant', - 'user', -]; - -const PureMessageSchema = z.object({ - content: z.string(), - attachments: z.array(z.string()).optional().nullable(), - params: z.record(z.any()).optional().nullable(), -}); - -export const PromptMessageSchema = PureMessageSchema.extend({ - role: z.enum(ChatMessageRole), -}).strict(); - -export type PromptMessage = z.infer; - -export type PromptParams = NonNullable; - -export const PromptConfigStrictSchema = z.object({ - // openai - jsonMode: z.boolean().nullable().optional(), - frequencyPenalty: z.number().nullable().optional(), - presencePenalty: z.number().nullable().optional(), - temperature: z.number().nullable().optional(), - topP: z.number().nullable().optional(), - maxTokens: z.number().nullable().optional(), - // fal - modelName: z.string().nullable().optional(), - loras: z - .array( - z.object({ path: z.string(), scale: z.number().nullable().optional() }) - ) - .nullable() - .optional(), - // google - audioTimestamp: z.boolean().nullable().optional(), -}); - -export const PromptConfigSchema = - PromptConfigStrictSchema.nullable().optional(); - -export type PromptConfig = z.infer; - export const ChatMessageSchema = PromptMessageSchema.extend({ id: z.string().optional(), createdAt: z.date(), }).strict(); - export type ChatMessage = z.infer; -export const SubmittedMessageSchema = PureMessageSchema.extend({ - sessionId: z.string(), - content: z.string().optional(), -}).strict(); - -export type SubmittedMessage = z.infer; - export const ChatHistorySchema = z .object({ sessionId: z.string(), action: z.string().nullable(), tokens: z.number(), - messages: z.array(PromptMessageSchema.or(ChatMessageSchema)), + messages: z.array(ChatMessageSchema), createdAt: z.date(), }) .strict(); export type ChatHistory = z.infer; +export const SubmittedMessageSchema = PureMessageSchema.extend({ + sessionId: z.string(), + content: z.string().optional(), +}).strict(); +export type SubmittedMessage = z.infer; + // ======== Chat Session ======== export interface ChatSessionOptions { @@ -154,142 +107,9 @@ export type ListHistoriesOptions = { withPrompt: boolean | undefined; }; -// ======== Provider Interface ======== - -export enum CopilotProviderType { - FAL = 'fal', - Google = 'google', - OpenAI = 'openai', - Perplexity = 'perplexity', - // only for test - Test = 'test', -} - -export enum CopilotCapability { - TextToText = 'text-to-text', - TextToEmbedding = 'text-to-embedding', - TextToImage = 'text-to-image', - ImageToImage = 'image-to-image', - ImageToText = 'image-to-text', -} - -const CopilotProviderOptionsSchema = z.object({ - signal: z.instanceof(AbortSignal).optional(), - user: z.string().optional(), -}); - -const CopilotChatOptionsSchema = CopilotProviderOptionsSchema.merge( - PromptConfigStrictSchema -).optional(); - -export type CopilotChatOptions = z.infer; - -const CopilotEmbeddingOptionsSchema = CopilotProviderOptionsSchema.extend({ - dimensions: z.number(), -}).optional(); - -export type CopilotEmbeddingOptions = z.infer< - typeof CopilotEmbeddingOptionsSchema ->; - -const CopilotImageOptionsSchema = CopilotProviderOptionsSchema.merge( - PromptConfigStrictSchema -) - .extend({ - seed: z.number().optional(), - }) - .optional(); - -export type CopilotImageOptions = z.infer; - export type CopilotContextFile = { id: string; // fileId created_at: number; // embedding status status: 'in_progress' | 'completed' | 'failed'; }; - -export interface CopilotProvider { - readonly type: CopilotProviderType; - getCapabilities(): CopilotCapability[]; - isModelAvailable(model: string): Promise; -} - -export interface CopilotTextToTextProvider extends CopilotProvider { - generateText( - messages: PromptMessage[], - model?: string, - options?: CopilotChatOptions - ): Promise; - generateTextStream( - messages: PromptMessage[], - model?: string, - options?: CopilotChatOptions - ): AsyncIterable; -} - -export interface CopilotTextToEmbeddingProvider extends CopilotProvider { - generateEmbedding( - messages: string[] | string, - model: string, - options?: CopilotEmbeddingOptions - ): Promise; -} - -export interface CopilotTextToImageProvider extends CopilotProvider { - generateImages( - messages: PromptMessage[], - model: string, - options?: CopilotImageOptions - ): Promise>; - generateImagesStream( - messages: PromptMessage[], - model?: string, - options?: CopilotImageOptions - ): AsyncIterable; -} - -export interface CopilotImageToTextProvider extends CopilotProvider { - generateText( - messages: PromptMessage[], - model: string, - options?: CopilotChatOptions - ): Promise; - generateTextStream( - messages: PromptMessage[], - model: string, - options?: CopilotChatOptions - ): AsyncIterable; -} - -export interface CopilotImageToImageProvider extends CopilotProvider { - generateImages( - messages: PromptMessage[], - model: string, - options?: CopilotImageOptions - ): Promise>; - generateImagesStream( - messages: PromptMessage[], - model?: string, - options?: CopilotImageOptions - ): AsyncIterable; -} - -export type CapabilityToCopilotProvider = { - [CopilotCapability.TextToText]: CopilotTextToTextProvider; - [CopilotCapability.TextToEmbedding]: CopilotTextToEmbeddingProvider; - [CopilotCapability.TextToImage]: CopilotTextToImageProvider; - [CopilotCapability.ImageToText]: CopilotImageToTextProvider; - [CopilotCapability.ImageToImage]: CopilotImageToImageProvider; -}; - -export type CopilotTextProvider = - | CopilotTextToTextProvider - | CopilotImageToTextProvider; -export type CopilotImageProvider = - | CopilotTextToImageProvider - | CopilotImageToImageProvider; -export type CopilotAllProvider = - | CopilotTextProvider - | CopilotImageProvider - | CopilotTextToEmbeddingProvider; diff --git a/packages/backend/server/src/plugins/copilot/workflow/executor/chat-image.ts b/packages/backend/server/src/plugins/copilot/workflow/executor/chat-image.ts index c86aa592ae..4bb2049e51 100644 --- a/packages/backend/server/src/plugins/copilot/workflow/executor/chat-image.ts +++ b/packages/backend/server/src/plugins/copilot/workflow/executor/chat-image.ts @@ -1,8 +1,11 @@ import { Injectable } from '@nestjs/common'; import { ChatPrompt, PromptService } from '../../prompt'; -import { CopilotProviderService } from '../../providers'; -import { CopilotChatOptions, CopilotImageProvider } from '../../types'; +import { + CopilotChatOptions, + CopilotImageProvider, + CopilotProviderFactory, +} from '../../providers'; import { WorkflowNodeData, WorkflowNodeType } from '../types'; import { NodeExecuteResult, NodeExecuteState, NodeExecutorType } from './types'; import { AutoRegisteredWorkflowExecutor } from './utils'; @@ -11,7 +14,7 @@ import { AutoRegisteredWorkflowExecutor } from './utils'; export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor { constructor( private readonly promptService: PromptService, - private readonly providerService: CopilotProviderService + private readonly providerFactory: CopilotProviderFactory ) { super(); } @@ -42,7 +45,7 @@ export class CopilotChatImageExecutor extends AutoRegisteredWorkflowExecutor { `Prompt ${data.promptName} not found when running workflow node ${data.name}` ); } - const provider = await this.providerService.getProviderByModel( + const provider = await this.providerFactory.getProviderByModel( prompt.model ); if (provider && 'generateImages' in provider) { diff --git a/packages/backend/server/src/plugins/copilot/workflow/executor/chat-text.ts b/packages/backend/server/src/plugins/copilot/workflow/executor/chat-text.ts index 609b4e73db..25b3aa567c 100644 --- a/packages/backend/server/src/plugins/copilot/workflow/executor/chat-text.ts +++ b/packages/backend/server/src/plugins/copilot/workflow/executor/chat-text.ts @@ -1,8 +1,11 @@ import { Injectable } from '@nestjs/common'; import { ChatPrompt, PromptService } from '../../prompt'; -import { CopilotProviderService } from '../../providers'; -import { CopilotChatOptions, CopilotTextProvider } from '../../types'; +import { + CopilotChatOptions, + CopilotProviderFactory, + CopilotTextProvider, +} from '../../providers'; import { WorkflowNodeData, WorkflowNodeType } from '../types'; import { NodeExecuteResult, NodeExecuteState, NodeExecutorType } from './types'; import { AutoRegisteredWorkflowExecutor } from './utils'; @@ -11,7 +14,7 @@ import { AutoRegisteredWorkflowExecutor } from './utils'; export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor { constructor( private readonly promptService: PromptService, - private readonly providerService: CopilotProviderService + private readonly providerFactory: CopilotProviderFactory ) { super(); } @@ -42,7 +45,7 @@ export class CopilotChatTextExecutor extends AutoRegisteredWorkflowExecutor { `Prompt ${data.promptName} not found when running workflow node ${data.name}` ); } - const provider = await this.providerService.getProviderByModel( + const provider = await this.providerFactory.getProviderByModel( prompt.model ); if (provider && 'generateText' in provider) { diff --git a/packages/backend/server/src/plugins/copilot/workflow/executor/types.ts b/packages/backend/server/src/plugins/copilot/workflow/executor/types.ts index 0245e4e3af..1f3309d648 100644 --- a/packages/backend/server/src/plugins/copilot/workflow/executor/types.ts +++ b/packages/backend/server/src/plugins/copilot/workflow/executor/types.ts @@ -1,4 +1,4 @@ -import { CopilotChatOptions } from '../../types'; +import { CopilotChatOptions } from '../../providers'; import type { WorkflowNode } from '../node'; import { WorkflowNodeData, WorkflowParams } from '../types'; diff --git a/packages/backend/server/src/plugins/copilot/workflow/node.ts b/packages/backend/server/src/plugins/copilot/workflow/node.ts index 01acbc7a05..45384060b6 100644 --- a/packages/backend/server/src/plugins/copilot/workflow/node.ts +++ b/packages/backend/server/src/plugins/copilot/workflow/node.ts @@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url'; import { Logger } from '@nestjs/common'; import Piscina from 'piscina'; -import { CopilotChatOptions } from '../types'; +import { CopilotChatOptions } from '../providers'; import type { NodeExecuteResult, NodeExecutor } from './executor'; import { getWorkflowExecutor, NodeExecuteState } from './executor'; import type { diff --git a/packages/backend/server/src/plugins/copilot/workflow/service.ts b/packages/backend/server/src/plugins/copilot/workflow/service.ts index 51393dd575..e58c893fef 100644 --- a/packages/backend/server/src/plugins/copilot/workflow/service.ts +++ b/packages/backend/server/src/plugins/copilot/workflow/service.ts @@ -1,6 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; -import { CopilotChatOptions } from '../types'; +import { CopilotChatOptions } from '../providers'; import { WorkflowGraphList } from './graph'; import { WorkflowNode } from './node'; import type { WorkflowGraph, WorkflowGraphInstances } from './types'; diff --git a/packages/backend/server/src/plugins/copilot/workflow/workflow.ts b/packages/backend/server/src/plugins/copilot/workflow/workflow.ts index 3671f0858e..03ee47b721 100644 --- a/packages/backend/server/src/plugins/copilot/workflow/workflow.ts +++ b/packages/backend/server/src/plugins/copilot/workflow/workflow.ts @@ -1,6 +1,6 @@ import { Logger } from '@nestjs/common'; -import { CopilotChatOptions } from '../types'; +import { CopilotChatOptions } from '../providers'; import { NodeExecuteState } from './executor'; import { WorkflowNode } from './node'; import type { WorkflowGraphInstances, WorkflowNodeState } from './types'; diff --git a/packages/backend/server/src/plugins/customerio/config.ts b/packages/backend/server/src/plugins/customerio/config.ts new file mode 100644 index 0000000000..c243f2a05c --- /dev/null +++ b/packages/backend/server/src/plugins/customerio/config.ts @@ -0,0 +1,22 @@ +import { defineModuleConfig } from '../../base'; + +declare global { + interface AppConfigSchema { + customerIo: { + enabled: boolean; + token: string; + }; + } +} + +defineModuleConfig('customerIo', { + enabled: { + desc: 'Enable customer.io integration', + default: false, + }, + token: { + desc: 'Customer.io token', + default: '', + schema: { type: 'string' }, + }, +}); diff --git a/packages/backend/server/src/plugins/customerio/index.ts b/packages/backend/server/src/plugins/customerio/index.ts new file mode 100644 index 0000000000..3a7fd4f507 --- /dev/null +++ b/packages/backend/server/src/plugins/customerio/index.ts @@ -0,0 +1,10 @@ +import './config'; + +import { Module } from '@nestjs/common'; + +import { CustomerIoService } from './service'; + +@Module({ + providers: [CustomerIoService], +}) +export class CustomerIoModule {} diff --git a/packages/backend/server/src/plugins/customerio/service.ts b/packages/backend/server/src/plugins/customerio/service.ts new file mode 100644 index 0000000000..4b1a1d0245 --- /dev/null +++ b/packages/backend/server/src/plugins/customerio/service.ts @@ -0,0 +1,72 @@ +import { Injectable } from '@nestjs/common'; + +import { Config, OnEvent } from '../../base'; + +@Injectable() +export class CustomerIoService { + #fetch: ((url: string, options?: RequestInit) => Promise) | null = + null; + constructor(private readonly config: Config) {} + + @OnEvent('config.init') + setup() { + const { enabled, token } = this.config.customerIo; + if (enabled && token) { + this.#fetch = (url, options) => { + return fetch(url, { + ...options, + headers: { + Authorization: `Basic ${token}`, + 'Content-Type': 'application/json', + ...options?.headers, + }, + }); + }; + } else { + this.#fetch = null; + } + } + + @OnEvent('config.changed') + onConfigChanged(event: Events['config.changed']) { + if (event.updates.customerIo) { + this.setup(); + } + } + + @OnEvent('user.created') + @OnEvent('user.updated') + async onUserUpdated(user: Events['user.updated'] | Events['user.created']) { + await this.#fetch?.( + `https://track.customer.io/api/v1/customers/${user.id}`, + { + method: 'PUT', + body: JSON.stringify({ + name: user.name, + email: user.email, + created_at: Number(user.createdAt) / 1000, + }), + } + ); + } + + @OnEvent('user.deleted') + async onUserDeleted(user: Events['user.deleted']) { + if (user.emailVerifiedAt) { + // suppress email if email is verified + await this.#fetch?.( + `https://track.customer.io/api/v1/customers/${user.email}/suppress`, + { + method: 'POST', + } + ); + } + + await this.#fetch?.( + `https://track.customer.io/api/v1/customers/${user.id}`, + { + method: 'DELETE', + } + ); + } +} diff --git a/packages/backend/server/src/plugins/gcloud/config.ts b/packages/backend/server/src/plugins/gcloud/config.ts deleted file mode 100644 index 0dd6bb7d4f..0000000000 --- a/packages/backend/server/src/plugins/gcloud/config.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { defineStartupConfig, ModuleConfig } from '../../base/config'; - -export interface GCloudConfig { - enabled: boolean; -} -declare module '../config' { - interface PluginsConfig { - gcloud: ModuleConfig; - } -} - -defineStartupConfig('plugins.gcloud', { - enabled: false, -}); diff --git a/packages/backend/server/src/plugins/gcloud/index.ts b/packages/backend/server/src/plugins/gcloud/index.ts index f2fbfe1a27..42241fdc4e 100644 --- a/packages/backend/server/src/plugins/gcloud/index.ts +++ b/packages/backend/server/src/plugins/gcloud/index.ts @@ -1,14 +1,10 @@ -import './config'; +import { Global, Module } from '@nestjs/common'; -import { Global } from '@nestjs/common'; - -import { Plugin } from '../registry'; import { GCloudLogging } from './logging'; import { GCloudMetrics } from './metrics'; @Global() -@Plugin({ - name: 'gcloud', +@Module({ imports: [GCloudMetrics, GCloudLogging], }) export class GCloudModule {} diff --git a/packages/backend/server/src/plugins/gcloud/logging/index.ts b/packages/backend/server/src/plugins/gcloud/logging/index.ts index 0109ee5ccf..2fe82ca326 100644 --- a/packages/backend/server/src/plugins/gcloud/logging/index.ts +++ b/packages/backend/server/src/plugins/gcloud/logging/index.ts @@ -1,12 +1,11 @@ -import { Global } from '@nestjs/common'; +import { Global, Module } from '@nestjs/common'; -import { OptionalModule } from '../../../base'; -import { loggerProvider } from './service'; +import { LoggerProvider } from './service'; @Global() -@OptionalModule({ - if: config => config.metrics.enabled, - overrides: [loggerProvider], +@Module({ + providers: [LoggerProvider], + exports: [LoggerProvider], }) export class GCloudLogging {} diff --git a/packages/backend/server/src/plugins/gcloud/logging/service.ts b/packages/backend/server/src/plugins/gcloud/logging/service.ts index 6305f058b4..1a648a33ee 100644 --- a/packages/backend/server/src/plugins/gcloud/logging/service.ts +++ b/packages/backend/server/src/plugins/gcloud/logging/service.ts @@ -9,7 +9,7 @@ const moreMetadata = format(info => { return info; }); -export const loggerProvider: Provider = { +export const LoggerProvider: Provider = { provide: LoggerProvide, useFactory: () => { const instance = createLogger({ diff --git a/packages/backend/server/src/plugins/gcloud/metrics.ts b/packages/backend/server/src/plugins/gcloud/metrics.ts index c95aced12d..ad06b6314c 100644 --- a/packages/backend/server/src/plugins/gcloud/metrics.ts +++ b/packages/backend/server/src/plugins/gcloud/metrics.ts @@ -1,7 +1,7 @@ import { MetricExporter } from '@google-cloud/opentelemetry-cloud-monitoring-exporter'; import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter'; import { GcpDetectorSync } from '@google-cloud/opentelemetry-resource-util'; -import { Global, Provider } from '@nestjs/common'; +import { Global, Injectable, Module, Provider } from '@nestjs/common'; import { getEnv } from '@opentelemetry/core'; import { Resource } from '@opentelemetry/resources'; import { @@ -14,9 +14,9 @@ import { SEMRESATTRS_K8S_POD_NAME, } from '@opentelemetry/semantic-conventions'; -import { OptionalModule } from '../../base'; import { OpentelemetryFactory } from '../../base/metrics'; +@Injectable() export class GCloudOpentelemetryFactory extends OpentelemetryFactory { override getResource(): Resource { const env = getEnv(); @@ -47,14 +47,14 @@ export class GCloudOpentelemetryFactory extends OpentelemetryFactory { } } -const factorProvider: Provider = { +const FactorProvider: Provider = { provide: OpentelemetryFactory, - useFactory: () => new GCloudOpentelemetryFactory(), + useClass: GCloudOpentelemetryFactory, }; @Global() -@OptionalModule({ - if: config => config.metrics.enabled, - overrides: [factorProvider], +@Module({ + providers: [FactorProvider], + exports: [FactorProvider], }) export class GCloudMetrics {} diff --git a/packages/backend/server/src/plugins/index.ts b/packages/backend/server/src/plugins/index.ts deleted file mode 100644 index 57fca3a307..0000000000 --- a/packages/backend/server/src/plugins/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import './captcha'; -import './copilot'; -import './gcloud'; -import './oauth'; -import './payment'; -import './storage'; -import './worker'; - -export { - enablePlugin, - REGISTERED_PLUGINS, - ENABLED_PLUGINS as USED_PLUGINS, -} from './registry'; diff --git a/packages/backend/server/src/plugins/license/index.ts b/packages/backend/server/src/plugins/license/index.ts index 854ee67a9e..615e2d2e73 100644 --- a/packages/backend/server/src/plugins/license/index.ts +++ b/packages/backend/server/src/plugins/license/index.ts @@ -1,10 +1,11 @@ -import { OptionalModule } from '../../base'; +import { Module } from '@nestjs/common'; + import { PermissionModule } from '../../core/permission'; import { QuotaModule } from '../../core/quota'; import { LicenseResolver } from './resolver'; import { LicenseService } from './service'; -@OptionalModule({ +@Module({ imports: [QuotaModule, PermissionModule], providers: [LicenseService, LicenseResolver], }) diff --git a/packages/backend/server/src/plugins/license/resolver.ts b/packages/backend/server/src/plugins/license/resolver.ts index 0cb78a4e91..75ebb01e2c 100644 --- a/packages/backend/server/src/plugins/license/resolver.ts +++ b/packages/backend/server/src/plugins/license/resolver.ts @@ -9,7 +9,7 @@ import { Resolver, } from '@nestjs/graphql'; -import { ActionForbidden, Config } from '../../base'; +import { UseNamedGuard } from '../../base'; import { CurrentUser } from '../../core/auth'; import { AccessController } from '../../core/permission'; import { WorkspaceType } from '../../core/workspaces'; @@ -34,10 +34,10 @@ export class License { expiredAt!: Date | null; } +@UseNamedGuard('selfhost') @Resolver(() => WorkspaceType) export class LicenseResolver { constructor( - private readonly config: Config, private readonly service: LicenseService, private readonly ac: AccessController ) {} @@ -51,13 +51,6 @@ export class LicenseResolver { @CurrentUser() user: CurrentUser, @Parent() workspace: WorkspaceType ): Promise { - // NOTE(@forehalo): - // we can't simply disable license resolver for non-selfhosted server - // it will make the gql codegen messed up. - if (!this.config.isSelfhosted) { - return null; - } - await this.ac .user(user.id) .workspace(workspace.id) @@ -71,10 +64,6 @@ export class LicenseResolver { @Args('workspaceId') workspaceId: string, @Args('license') license: string ) { - if (!this.config.isSelfhosted) { - throw new ActionForbidden(); - } - await this.ac .user(user.id) .workspace(workspaceId) @@ -88,10 +77,6 @@ export class LicenseResolver { @CurrentUser() user: CurrentUser, @Args('workspaceId') workspaceId: string ) { - if (!this.config.isSelfhosted) { - throw new ActionForbidden(); - } - await this.ac .user(user.id) .workspace(workspaceId) @@ -105,10 +90,6 @@ export class LicenseResolver { @CurrentUser() user: CurrentUser, @Args('workspaceId') workspaceId: string ) { - if (!this.config.isSelfhosted) { - throw new ActionForbidden(); - } - await this.ac .user(user.id) .workspace(workspaceId) diff --git a/packages/backend/server/src/plugins/license/service.ts b/packages/backend/server/src/plugins/license/service.ts index 9837550cda..32faf110cf 100644 --- a/packages/backend/server/src/plugins/license/service.ts +++ b/packages/backend/server/src/plugins/license/service.ts @@ -1,9 +1,8 @@ -import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { InstalledLicense, PrismaClient } from '@prisma/client'; import { - Config, EventBus, InternalServerError, LicenseNotFound, @@ -22,35 +21,22 @@ interface License { } @Injectable() -export class LicenseService implements OnModuleInit { +export class LicenseService { private readonly logger = new Logger(LicenseService.name); constructor( - private readonly config: Config, private readonly db: PrismaClient, private readonly event: EventBus, private readonly models: Models ) {} - async onModuleInit() { - if (this.config.isSelfhosted) { - this.event.on( - 'workspace.subscription.activated', - this.onWorkspaceSubscriptionUpdated - ); - this.event.on( - 'workspace.subscription.canceled', - this.onWorkspaceSubscriptionCanceled - ); - } - } - - private readonly onWorkspaceSubscriptionUpdated = async ({ + @OnEvent('workspace.subscription.activated') + async onWorkspaceSubscriptionUpdated({ workspaceId, plan, recurring, quantity, - }: Events['workspace.subscription.activated']) => { + }: Events['workspace.subscription.activated']) { switch (plan) { case SubscriptionPlan.SelfHostedTeam: await this.models.workspaceFeature.add( @@ -66,12 +52,13 @@ export class LicenseService implements OnModuleInit { default: break; } - }; + } - private readonly onWorkspaceSubscriptionCanceled = async ({ + @OnEvent('workspace.subscription.canceled') + async onWorkspaceSubscriptionCanceled({ workspaceId, plan, - }: Events['workspace.subscription.canceled']) => { + }: Events['workspace.subscription.canceled']) { switch (plan) { case SubscriptionPlan.SelfHostedTeam: await this.models.workspaceFeature.remove(workspaceId, 'team_plan_v1'); @@ -79,7 +66,7 @@ export class LicenseService implements OnModuleInit { default: break; } - }; + } async getLicense(workspaceId: string) { return this.db.installedLicense.findUnique({ @@ -201,10 +188,6 @@ export class LicenseService implements OnModuleInit { @OnEvent('workspace.members.updated') async updateTeamSeats(payload: Events['workspace.members.updated']) { - if (!this.config.isSelfhosted) { - return; - } - const { workspaceId, count } = payload; const license = await this.db.installedLicense.findUnique({ @@ -251,12 +234,8 @@ export class LicenseService implements OnModuleInit { throw new Error('Timeout checking seat update result.'); } - @Cron(CronExpression.EVERY_10_MINUTES) + @Cron(CronExpression.EVERY_10_MINUTES, { disabled: !env.selfhosted }) async licensesHealthCheck() { - if (!this.config.isSelfhosted) { - return; - } - const licenses = await this.db.installedLicense.findMany({ where: { validatedAt: { diff --git a/packages/backend/server/src/plugins/oauth/config.ts b/packages/backend/server/src/plugins/oauth/config.ts index f3bff3a90b..11bb74748f 100644 --- a/packages/backend/server/src/plugins/oauth/config.ts +++ b/packages/backend/server/src/plugins/oauth/config.ts @@ -1,4 +1,4 @@ -import { defineStartupConfig, ModuleConfig } from '../../base/config'; +import { defineModuleConfig, JSONSchema } from '../../base'; export interface OAuthProviderConfig { clientId: string; @@ -23,23 +23,53 @@ export enum OAuthProviderName { GitHub = 'github', OIDC = 'oidc', } - -type OAuthProviderConfigMapping = { - [OAuthProviderName.Google]: OAuthProviderConfig; - [OAuthProviderName.GitHub]: OAuthProviderConfig; - [OAuthProviderName.OIDC]: OAuthOIDCProviderConfig; -}; - -export interface OAuthConfig { - providers: Partial; -} - -declare module '../config' { - interface PluginsConfig { - oauth: ModuleConfig; +declare global { + interface AppConfigSchema { + oauth: { + providers: { + [OAuthProviderName.Google]: ConfigItem; + [OAuthProviderName.GitHub]: ConfigItem; + [OAuthProviderName.OIDC]: ConfigItem; + }; + }; } } -defineStartupConfig('plugins.oauth', { - providers: {}, +const schema: JSONSchema = { + type: 'object', + properties: { + clientId: { type: 'string' }, + clientSecret: { type: 'string' }, + args: { type: 'object' }, + }, +}; + +defineModuleConfig('oauth', { + 'providers.google': { + desc: 'Google OAuth provider config', + default: { + clientId: '', + clientSecret: '', + }, + schema, + link: 'https://developers.google.com/identity/protocols/oauth2/web-server', + }, + 'providers.github': { + desc: 'GitHub OAuth provider config', + default: { + clientId: '', + clientSecret: '', + }, + schema, + link: 'https://docs.github.com/en/apps/oauth-apps', + }, + 'providers.oidc': { + desc: 'OIDC OAuth provider config', + default: { + clientId: '', + clientSecret: '', + issuer: '', + args: {}, + }, + }, }); diff --git a/packages/backend/server/src/plugins/oauth/controller.ts b/packages/backend/server/src/plugins/oauth/controller.ts index 5c9a0ec217..d3eb241e02 100644 --- a/packages/backend/server/src/plugins/oauth/controller.ts +++ b/packages/backend/server/src/plugins/oauth/controller.ts @@ -24,8 +24,8 @@ import { import { AuthService, Public } from '../../core/auth'; import { Models } from '../../models'; import { OAuthProviderName } from './config'; +import { OAuthProviderFactory } from './factory'; import { OAuthAccount, Tokens } from './providers/def'; -import { OAuthProviderFactory } from './register'; import { OAuthService } from './service'; @Controller('/api/oauth') diff --git a/packages/backend/server/src/plugins/oauth/factory.ts b/packages/backend/server/src/plugins/oauth/factory.ts new file mode 100644 index 0000000000..2e30fb4c73 --- /dev/null +++ b/packages/backend/server/src/plugins/oauth/factory.ts @@ -0,0 +1,35 @@ +import { Injectable, Logger } from '@nestjs/common'; + +import { ServerFeature, ServerService } from '../../core'; +import { OAuthProviderName } from './config'; +import type { OAuthProvider } from './providers/def'; + +@Injectable() +export class OAuthProviderFactory { + constructor(private readonly server: ServerService) {} + + private readonly logger = new Logger(OAuthProviderFactory.name); + readonly #providers = new Map(); + + get providers() { + return Array.from(this.#providers.keys()); + } + + get(name: OAuthProviderName): OAuthProvider | undefined { + return this.#providers.get(name); + } + + register(provider: OAuthProvider) { + this.#providers.set(provider.provider, provider); + this.logger.log(`OAuth provider [${provider.provider}] registered.`); + this.server.enableFeature(ServerFeature.OAuth); + } + + unregister(provider: OAuthProvider) { + this.#providers.delete(provider.provider); + this.logger.log(`OAuth provider [${provider.provider}] unregistered.`); + if (this.#providers.size === 0) { + this.server.disableFeature(ServerFeature.OAuth); + } + } +} diff --git a/packages/backend/server/src/plugins/oauth/index.ts b/packages/backend/server/src/plugins/oauth/index.ts index abfc3aab2e..14f5f90146 100644 --- a/packages/backend/server/src/plugins/oauth/index.ts +++ b/packages/backend/server/src/plugins/oauth/index.ts @@ -1,18 +1,18 @@ import './config'; +import { Module } from '@nestjs/common'; + +import { ServerConfigModule } from '../../core'; import { AuthModule } from '../../core/auth'; -import { ServerFeature } from '../../core/config'; import { UserModule } from '../../core/user'; -import { Plugin } from '../registry'; import { OAuthController } from './controller'; +import { OAuthProviderFactory } from './factory'; import { OAuthProviders } from './providers'; -import { OAuthProviderFactory } from './register'; import { OAuthResolver } from './resolver'; import { OAuthService } from './service'; -@Plugin({ - name: 'oauth', - imports: [AuthModule, UserModule], +@Module({ + imports: [AuthModule, UserModule, ServerConfigModule], providers: [ OAuthProviderFactory, OAuthService, @@ -20,7 +20,5 @@ import { OAuthService } from './service'; ...OAuthProviders, ], controllers: [OAuthController], - contributesTo: ServerFeature.OAuth, - if: config => config.flavor.graphql && !!config.plugins.oauth, }) export class OAuthModule {} diff --git a/packages/backend/server/src/plugins/oauth/providers/def.ts b/packages/backend/server/src/plugins/oauth/providers/def.ts index 417df267c6..a1ed8c937c 100644 --- a/packages/backend/server/src/plugins/oauth/providers/def.ts +++ b/packages/backend/server/src/plugins/oauth/providers/def.ts @@ -1,4 +1,8 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; + +import { Config, OnEvent } from '../../../base'; import { OAuthProviderName } from '../config'; +import { OAuthProviderFactory } from '../factory'; export interface OAuthAccount { id: string; @@ -13,9 +17,42 @@ export interface Tokens { expiresAt?: Date; } +@Injectable() export abstract class OAuthProvider { abstract provider: OAuthProviderName; abstract getAuthUrl(state: string): string; abstract getToken(code: string): Promise; abstract getUser(token: string): Promise; + + protected readonly logger = new Logger(this.constructor.name); + @Inject() private readonly factory!: OAuthProviderFactory; + @Inject() private readonly AFFiNEConfig!: Config; + + get config() { + return this.AFFiNEConfig.oauth.providers[this.provider]; + } + + get configured() { + return this.config && this.config.clientId && this.config.clientSecret; + } + + @OnEvent('config.init') + onConfigInit() { + this.setup(); + } + + @OnEvent('config.changed') + onConfigUpdated(event: Events['config.changed']) { + if ('oauth' in event.updates) { + this.setup(); + } + } + + protected setup() { + if (this.configured) { + this.factory.register(this); + } else { + this.factory.unregister(this); + } + } } diff --git a/packages/backend/server/src/plugins/oauth/providers/github.ts b/packages/backend/server/src/plugins/oauth/providers/github.ts index bff59262f3..25f7f36bd6 100644 --- a/packages/backend/server/src/plugins/oauth/providers/github.ts +++ b/packages/backend/server/src/plugins/oauth/providers/github.ts @@ -1,8 +1,8 @@ import { Injectable } from '@nestjs/common'; -import { Config, InvalidOauthCallbackCode, URLHelper } from '../../../base'; +import { InvalidOauthCallbackCode, URLHelper } from '../../../base'; import { OAuthProviderName } from '../config'; -import { AutoRegisteredOAuthProvider } from '../register'; +import { OAuthProvider } from './def'; interface AuthTokenResponse { access_token: string; @@ -18,13 +18,10 @@ export interface UserInfo { } @Injectable() -export class GithubOAuthProvider extends AutoRegisteredOAuthProvider { +export class GithubOAuthProvider extends OAuthProvider { provider = OAuthProviderName.GitHub; - constructor( - protected readonly AFFiNEConfig: Config, - private readonly url: URLHelper - ) { + constructor(private readonly url: URLHelper) { super(); } diff --git a/packages/backend/server/src/plugins/oauth/providers/google.ts b/packages/backend/server/src/plugins/oauth/providers/google.ts index 8bb2f15360..9a48f16f13 100644 --- a/packages/backend/server/src/plugins/oauth/providers/google.ts +++ b/packages/backend/server/src/plugins/oauth/providers/google.ts @@ -1,8 +1,8 @@ import { Injectable } from '@nestjs/common'; -import { Config, InvalidOauthCallbackCode, URLHelper } from '../../../base'; +import { InvalidOauthCallbackCode, URLHelper } from '../../../base'; import { OAuthProviderName } from '../config'; -import { AutoRegisteredOAuthProvider } from '../register'; +import { OAuthProvider } from './def'; interface GoogleOAuthTokenResponse { access_token: string; @@ -20,13 +20,10 @@ export interface UserInfo { } @Injectable() -export class GoogleOAuthProvider extends AutoRegisteredOAuthProvider { +export class GoogleOAuthProvider extends OAuthProvider { override provider = OAuthProviderName.Google; - constructor( - protected readonly AFFiNEConfig: Config, - private readonly url: URLHelper - ) { + constructor(private readonly url: URLHelper) { super(); } diff --git a/packages/backend/server/src/plugins/oauth/providers/oidc.ts b/packages/backend/server/src/plugins/oauth/providers/oidc.ts index 36a34204fa..0ddeba3486 100644 --- a/packages/backend/server/src/plugins/oauth/providers/oidc.ts +++ b/packages/backend/server/src/plugins/oauth/providers/oidc.ts @@ -1,14 +1,13 @@ -import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { z } from 'zod'; -import { Config, URLHelper } from '../../../base'; +import { URLHelper } from '../../../base'; import { OAuthOIDCProviderConfig, OAuthProviderName, OIDCArgs, } from '../config'; -import { AutoRegisteredOAuthProvider } from '../register'; -import { OAuthAccount, Tokens } from './def'; +import { OAuthAccount, OAuthProvider, Tokens } from './def'; const OIDCTokenSchema = z.object({ access_token: z.string(), @@ -163,26 +162,26 @@ class OIDCClient { } @Injectable() -export class OIDCProvider - extends AutoRegisteredOAuthProvider - implements OnModuleInit -{ +export class OIDCProvider extends OAuthProvider { override provider = OAuthProviderName.OIDC; private client: OIDCClient | null = null; - constructor( - protected readonly AFFiNEConfig: Config, - private readonly url: URLHelper - ) { + constructor(private readonly url: URLHelper) { super(); } - // eslint-disable-next-line @typescript-eslint/no-misused-promises - override async onModuleInit() { - const config = this.optionalConfig as OAuthOIDCProviderConfig; - if (config && config.issuer && config.clientId && config.clientSecret) { - this.client = await OIDCClient.create(config, this.url); - super.onModuleInit(); + protected override setup() { + super.setup(); + if (this.configured) { + OIDCClient.create(this.config as OAuthOIDCProviderConfig, this.url) + .then(client => { + this.client = client; + }) + .catch(e => { + this.logger.error('Failed to create OIDC client', e); + }); + } else { + this.client = null; } } diff --git a/packages/backend/server/src/plugins/oauth/register.ts b/packages/backend/server/src/plugins/oauth/register.ts deleted file mode 100644 index a1399b68c7..0000000000 --- a/packages/backend/server/src/plugins/oauth/register.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; - -import { Config } from '../../base'; -import { OAuthProviderName } from './config'; -import { OAuthProvider } from './providers/def'; - -const PROVIDERS: Map = new Map(); - -export function registerOAuthProvider( - name: OAuthProviderName, - provider: OAuthProvider -) { - PROVIDERS.set(name, provider); -} - -@Injectable() -export class OAuthProviderFactory { - get providers() { - return Array.from(PROVIDERS.keys()); - } - - get(name: OAuthProviderName): OAuthProvider | undefined { - return PROVIDERS.get(name); - } -} - -export abstract class AutoRegisteredOAuthProvider - extends OAuthProvider - implements OnModuleInit -{ - protected abstract AFFiNEConfig: Config; - - get optionalConfig() { - return this.AFFiNEConfig.plugins.oauth?.providers?.[this.provider]; - } - - get config() { - const config = this.optionalConfig; - - if (!config) { - throw new Error( - `OAuthProvider Config should not be used before registered` - ); - } - - return config; - } - - onModuleInit() { - const config = this.optionalConfig; - if (config && config.clientId && config.clientSecret) { - registerOAuthProvider(this.provider, this); - new Logger(`OAuthProvider:${this.provider}`).log( - 'OAuth provider registered.' - ); - } - } -} diff --git a/packages/backend/server/src/plugins/oauth/resolver.ts b/packages/backend/server/src/plugins/oauth/resolver.ts index 02cd5f9e0e..b69d8f6e13 100644 --- a/packages/backend/server/src/plugins/oauth/resolver.ts +++ b/packages/backend/server/src/plugins/oauth/resolver.ts @@ -2,7 +2,7 @@ import { registerEnumType, ResolveField, Resolver } from '@nestjs/graphql'; import { ServerConfigType } from '../../core/config/types'; import { OAuthProviderName } from './config'; -import { OAuthProviderFactory } from './register'; +import { OAuthProviderFactory } from './factory'; registerEnumType(OAuthProviderName, { name: 'OAuthProviderType' }); diff --git a/packages/backend/server/src/plugins/oauth/service.ts b/packages/backend/server/src/plugins/oauth/service.ts index 251425f663..b6f06b6053 100644 --- a/packages/backend/server/src/plugins/oauth/service.ts +++ b/packages/backend/server/src/plugins/oauth/service.ts @@ -4,7 +4,7 @@ import { Injectable } from '@nestjs/common'; import { SessionCache } from '../../base'; import { OAuthProviderName } from './config'; -import { OAuthProviderFactory } from './register'; +import { OAuthProviderFactory } from './factory'; const OAUTH_STATE_KEY = 'OAUTH_STATE'; diff --git a/packages/backend/server/src/plugins/payment/config.ts b/packages/backend/server/src/plugins/payment/config.ts index 114c5ed04c..90abc672d2 100644 --- a/packages/backend/server/src/plugins/payment/config.ts +++ b/packages/backend/server/src/plugins/payment/config.ts @@ -1,10 +1,6 @@ import type { Stripe } from 'stripe'; -import { - defineRuntimeConfig, - defineStartupConfig, - ModuleConfig, -} from '../../base/config'; +import { defineModuleConfig } from '../../base'; export interface PaymentStartupConfig { stripe?: { @@ -19,16 +15,40 @@ export interface PaymentRuntimeConfig { showLifetimePrice: boolean; } -declare module '../config' { - interface PluginsConfig { - payment: ModuleConfig; +declare global { + interface AppConfigSchema { + payment: { + enabled: boolean; + showLifetimePrice: boolean; + apiKey: string; + webhookKey: string; + stripe: ConfigItem<{} & Stripe.StripeConfig>; + }; } } -defineStartupConfig('plugins.payment', {}); -defineRuntimeConfig('plugins.payment', { +defineModuleConfig('payment', { + enabled: { + desc: 'Whether enable payment plugin', + default: false, + }, showLifetimePrice: { desc: 'Whether enable lifetime price and allow user to pay for it.', default: true, }, + apiKey: { + desc: 'Stripe API key to enable payment service.', + default: '', + env: 'STRIPE_API_KEY', + }, + webhookKey: { + desc: 'Stripe webhook key to enable payment service.', + default: '', + env: 'STRIPE_WEBHOOK_KEY', + }, + stripe: { + desc: 'Stripe API keys', + default: {}, + link: 'https://docs.stripe.com/api', + }, }); diff --git a/packages/backend/server/src/plugins/payment/controller.ts b/packages/backend/server/src/plugins/payment/controller.ts index 2225aef444..03074d39fe 100644 --- a/packages/backend/server/src/plugins/payment/controller.ts +++ b/packages/backend/server/src/plugins/payment/controller.ts @@ -1,37 +1,32 @@ -import assert from 'node:assert'; - import type { RawBodyRequest } from '@nestjs/common'; import { Controller, Logger, Post, Req } from '@nestjs/common'; import type { Request } from 'express'; -import Stripe from 'stripe'; import { Config, EventBus, InternalServerError } from '../../base'; import { Public } from '../../core/auth'; +import { StripeFactory } from './stripe'; @Controller('/api/stripe') export class StripeWebhookController { - private readonly webhookKey: string; private readonly logger = new Logger(StripeWebhookController.name); constructor( - config: Config, - private readonly stripe: Stripe, + private readonly config: Config, + private readonly stripeProvider: StripeFactory, private readonly event: EventBus - ) { - assert(config.plugins.payment.stripe); - this.webhookKey = config.plugins.payment.stripe.keys.webhookKey; - } + ) {} @Public() @Post('/webhook') async handleWebhook(@Req() req: RawBodyRequest) { + const webhookKey = this.config.payment.webhookKey; // Retrieve the event by verifying the signature using the raw body and secret. const signature = req.headers['stripe-signature']; try { - const event = this.stripe.webhooks.constructEvent( + const event = this.stripeProvider.stripe.webhooks.constructEvent( req.rawBody ?? '', signature ?? '', - this.webhookKey + webhookKey ); this.logger.debug( diff --git a/packages/backend/server/src/plugins/payment/index.ts b/packages/backend/server/src/plugins/payment/index.ts index cdcc9520cd..8aaa8b1a50 100644 --- a/packages/backend/server/src/plugins/payment/index.ts +++ b/packages/backend/server/src/plugins/payment/index.ts @@ -1,13 +1,14 @@ import './config'; -import { ServerFeature } from '../../core/config'; +import { Module } from '@nestjs/common'; + +import { ServerConfigModule } from '../../core'; import { FeatureModule } from '../../core/features'; import { MailModule } from '../../core/mail'; import { PermissionModule } from '../../core/permission'; import { QuotaModule } from '../../core/quota'; import { UserModule } from '../../core/user'; import { WorkspaceModule } from '../../core/workspaces'; -import { Plugin } from '../registry'; import { StripeWebhookController } from './controller'; import { SubscriptionCronJobs } from './cron'; import { LicenseController } from './license/controller'; @@ -23,11 +24,10 @@ import { WorkspaceSubscriptionResolver, } from './resolver'; import { SubscriptionService } from './service'; -import { StripeProvider } from './stripe'; +import { StripeFactory, StripeProvider } from './stripe'; import { StripeWebhook } from './webhook'; -@Plugin({ - name: 'payment', +@Module({ imports: [ FeatureModule, QuotaModule, @@ -35,8 +35,10 @@ import { StripeWebhook } from './webhook'; PermissionModule, WorkspaceModule, MailModule, + ServerConfigModule, ], providers: [ + StripeFactory, StripeProvider, SubscriptionService, SubscriptionResolver, @@ -50,11 +52,5 @@ import { StripeWebhook } from './webhook'; QuotaOverride, ], controllers: [StripeWebhookController, LicenseController], - requires: [ - 'plugins.payment.stripe.keys.APIKey', - 'plugins.payment.stripe.keys.webhookKey', - ], - contributesTo: ServerFeature.Payment, - if: config => config.flavor.graphql, }) export class PaymentModule {} diff --git a/packages/backend/server/src/plugins/payment/license/controller.ts b/packages/backend/server/src/plugins/payment/license/controller.ts index adef69d123..80a8b95cc2 100644 --- a/packages/backend/server/src/plugins/payment/license/controller.ts +++ b/packages/backend/server/src/plugins/payment/license/controller.ts @@ -26,6 +26,7 @@ import { import { Public } from '../../../core/auth'; import { SelfhostTeamSubscriptionManager } from '../manager/selfhost'; import { SubscriptionService } from '../service'; +import { StripeFactory } from '../stripe'; import { SubscriptionPlan, SubscriptionRecurring, @@ -53,7 +54,7 @@ export class LicenseController { private readonly mutex: Mutex, private readonly subscription: SubscriptionService, private readonly manager: SelfhostTeamSubscriptionManager, - private readonly stripe: Stripe + private readonly stripeProvider: StripeFactory ) {} @Post('/:license/activate') @@ -238,18 +239,20 @@ export class LicenseController { throw new LicenseNotFound(); } - const subscriptionData = await this.stripe.subscriptions.retrieve( - subscription.stripeSubscriptionId, - { - expand: ['customer'], - } - ); + const subscriptionData = + await this.stripeProvider.stripe.subscriptions.retrieve( + subscription.stripeSubscriptionId, + { + expand: ['customer'], + } + ); const customer = subscriptionData.customer as Stripe.Customer; try { - const portal = await this.stripe.billingPortal.sessions.create({ - customer: customer.id, - }); + const portal = + await this.stripeProvider.stripe.billingPortal.sessions.create({ + customer: customer.id, + }); return { url: portal.url }; } catch (e) { diff --git a/packages/backend/server/src/plugins/payment/manager/common.ts b/packages/backend/server/src/plugins/payment/manager/common.ts index c7661e4a5c..352cb53ee6 100644 --- a/packages/backend/server/src/plugins/payment/manager/common.ts +++ b/packages/backend/server/src/plugins/payment/manager/common.ts @@ -4,6 +4,7 @@ import { z } from 'zod'; import { UserNotFound } from '../../../base'; import { ScheduleManager } from '../schedule'; +import { StripeFactory } from '../stripe'; import { encodeLookupKey, KnownStripeInvoice, @@ -55,12 +56,16 @@ export const CheckoutParams = z.object({ }); export abstract class SubscriptionManager { - protected readonly scheduleManager = new ScheduleManager(this.stripe); + protected readonly scheduleManager = new ScheduleManager(this.stripeProvider); constructor( - protected readonly stripe: Stripe, + protected readonly stripeProvider: StripeFactory, protected readonly db: PrismaClient ) {} + get stripe() { + return this.stripeProvider.stripe; + } + abstract filterPrices( prices: KnownStripePrice[], customer?: UserStripeCustomer diff --git a/packages/backend/server/src/plugins/payment/manager/selfhost.ts b/packages/backend/server/src/plugins/payment/manager/selfhost.ts index 53aa786cfb..ba1e0a335e 100644 --- a/packages/backend/server/src/plugins/payment/manager/selfhost.ts +++ b/packages/backend/server/src/plugins/payment/manager/selfhost.ts @@ -3,11 +3,11 @@ import { randomUUID } from 'node:crypto'; import { Injectable } from '@nestjs/common'; import { PrismaClient, UserStripeCustomer } from '@prisma/client'; import { pick } from 'lodash-es'; -import Stripe from 'stripe'; import { z } from 'zod'; import { SubscriptionPlanNotFound, URLHelper } from '../../../base'; import { Mailer } from '../../../core/mail'; +import { StripeFactory } from '../stripe'; import { KnownStripeInvoice, KnownStripePrice, @@ -43,12 +43,12 @@ export const SelfhostTeamSubscriptionIdentity = z.object({ @Injectable() export class SelfhostTeamSubscriptionManager extends SubscriptionManager { constructor( - stripe: Stripe, + stripeProvider: StripeFactory, db: PrismaClient, private readonly url: URLHelper, private readonly mailer: Mailer ) { - super(stripe, db); + super(stripeProvider, db); } filterPrices( diff --git a/packages/backend/server/src/plugins/payment/manager/user.ts b/packages/backend/server/src/plugins/payment/manager/user.ts index 403d06c850..d7a2a713f8 100644 --- a/packages/backend/server/src/plugins/payment/manager/user.ts +++ b/packages/backend/server/src/plugins/payment/manager/user.ts @@ -5,17 +5,18 @@ import Stripe from 'stripe'; import { z } from 'zod'; import { + Config, EventBus, InternalServerError, InvalidCheckoutParameters, Mutex, - Runtime, SubscriptionAlreadyExists, SubscriptionPlanNotFound, TooManyRequest, URLHelper, } from '../../../base'; import { EarlyAccessType, FeatureService } from '../../../core/features'; +import { StripeFactory } from '../stripe'; import { CouponType, KnownStripeInvoice, @@ -53,15 +54,15 @@ export const UserSubscriptionCheckoutArgs = z.object({ @Injectable() export class UserSubscriptionManager extends SubscriptionManager { constructor( - stripe: Stripe, + stripeProvider: StripeFactory, db: PrismaClient, - private readonly runtime: Runtime, + private readonly config: Config, private readonly feature: FeatureService, private readonly event: EventBus, private readonly url: URLHelper, private readonly mutex: Mutex ) { - super(stripe, db); + super(stripeProvider, db); } async filterPrices( @@ -588,7 +589,7 @@ export class UserSubscriptionManager extends SubscriptionManager { { proEarlyAccess, proSubscribed, onetime }: PriceStrategyStatus ) { if (lookupKey.recurring === SubscriptionRecurring.Lifetime) { - return this.runtime.fetch('plugins.payment/showLifetimePrice'); + return this.config.payment.showLifetimePrice; } if (lookupKey.variant === SubscriptionVariant.Onetime) { diff --git a/packages/backend/server/src/plugins/payment/manager/workspace.ts b/packages/backend/server/src/plugins/payment/manager/workspace.ts index 40fbeb0a44..f46fa2f839 100644 --- a/packages/backend/server/src/plugins/payment/manager/workspace.ts +++ b/packages/backend/server/src/plugins/payment/manager/workspace.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { PrismaClient, UserStripeCustomer } from '@prisma/client'; import { omit, pick } from 'lodash-es'; -import Stripe from 'stripe'; import { z } from 'zod'; import { @@ -12,6 +11,7 @@ import { URLHelper, } from '../../../base'; import { Models } from '../../../models'; +import { StripeFactory } from '../stripe'; import { KnownStripeInvoice, KnownStripePrice, @@ -46,13 +46,13 @@ export const WorkspaceSubscriptionCheckoutArgs = z.object({ @Injectable() export class WorkspaceSubscriptionManager extends SubscriptionManager { constructor( - stripe: Stripe, + stripeProvider: StripeFactory, db: PrismaClient, private readonly url: URLHelper, private readonly event: EventBus, private readonly models: Models ) { - super(stripe, db); + super(stripeProvider, db); } filterPrices( diff --git a/packages/backend/server/src/plugins/payment/schedule.ts b/packages/backend/server/src/plugins/payment/schedule.ts index b4c2abb35f..38df00868a 100644 --- a/packages/backend/server/src/plugins/payment/schedule.ts +++ b/packages/backend/server/src/plugins/payment/schedule.ts @@ -1,15 +1,24 @@ import { Injectable, Logger } from '@nestjs/common'; import Stripe from 'stripe'; +import { StripeFactory } from './stripe'; + @Injectable() export class ScheduleManager { private _schedule: Stripe.SubscriptionSchedule | null = null; private readonly logger = new Logger(ScheduleManager.name); - constructor(private readonly stripe: Stripe) {} + constructor(private readonly stripeProvider: StripeFactory) {} - static create(stripe: Stripe, schedule?: Stripe.SubscriptionSchedule) { - const manager = new ScheduleManager(stripe); + get stripe() { + return this.stripeProvider.stripe; + } + + static create( + stripeProvider: StripeFactory, + schedule?: Stripe.SubscriptionSchedule + ) { + const manager = new ScheduleManager(stripeProvider); if (schedule) { manager._schedule = schedule; } @@ -56,9 +65,9 @@ export class ScheduleManager { return undefined; }); - return ScheduleManager.create(this.stripe, s); + return ScheduleManager.create(this.stripeProvider, s); } else { - return ScheduleManager.create(this.stripe, schedule); + return ScheduleManager.create(this.stripeProvider, schedule); } } diff --git a/packages/backend/server/src/plugins/payment/service.ts b/packages/backend/server/src/plugins/payment/service.ts index 178ff8bacc..f72852255d 100644 --- a/packages/backend/server/src/plugins/payment/service.ts +++ b/packages/backend/server/src/plugins/payment/service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import type { User, UserStripeCustomer } from '@prisma/client'; import { PrismaClient } from '@prisma/client'; import Stripe from 'stripe'; @@ -7,14 +7,12 @@ import { z } from 'zod'; import { ActionForbidden, CantUpdateOnetimePaymentSubscription, - Config, CustomerPortalCreateFailed, InternalServerError, InvalidCheckoutParameters, InvalidLicenseSessionId, InvalidSubscriptionParameters, LicenseRevealed, - Mutex, OnEvent, SameSubscriptionRecurring, SubscriptionExpired, @@ -46,9 +44,8 @@ import { SelfhostTeamSubscriptionManager, } from './manager/selfhost'; import { ScheduleManager } from './schedule'; +import { StripeFactory } from './stripe'; import { - decodeLookupKey, - DEFAULT_PRICES, KnownStripeInvoice, KnownStripePrice, KnownStripeSubscription, @@ -57,7 +54,6 @@ import { SubscriptionPlan, SubscriptionRecurring, SubscriptionStatus, - SubscriptionVariant, } from './types'; export const CheckoutExtraArgs = z.union([ @@ -75,24 +71,22 @@ export const SubscriptionIdentity = z.union([ export { CheckoutParams }; @Injectable() -export class SubscriptionService implements OnApplicationBootstrap { +export class SubscriptionService { private readonly logger = new Logger(SubscriptionService.name); - private readonly scheduleManager = new ScheduleManager(this.stripe); + private readonly scheduleManager = new ScheduleManager(this.stripeProvider); constructor( - private readonly config: Config, - private readonly stripe: Stripe, + private readonly stripeProvider: StripeFactory, private readonly db: PrismaClient, private readonly feature: FeatureService, private readonly models: Models, private readonly userManager: UserSubscriptionManager, private readonly workspaceManager: WorkspaceSubscriptionManager, - private readonly selfhostManager: SelfhostTeamSubscriptionManager, - private readonly mutex: Mutex + private readonly selfhostManager: SelfhostTeamSubscriptionManager ) {} - async onApplicationBootstrap() { - await this.initStripeProducts(); + get stripe() { + return this.stripeProvider.stripe; } private select(plan: SubscriptionPlan): SubscriptionManager { @@ -132,8 +126,8 @@ export class SubscriptionService implements OnApplicationBootstrap { const { plan, recurring, variant } = params; if ( - this.config.deploy && - this.config.affine.canary && + env.namespaces.canary && + env.prod && args.user && !this.feature.isStaff(args.user.email) ) { @@ -683,72 +677,4 @@ export class SubscriptionService implements OnApplicationBootstrap { throw new InvalidSubscriptionParameters(); } } - - private async initStripeProducts() { - // only init stripe products in dev mode or canary deployment - if ( - (this.config.deploy && !this.config.affine.canary) || - !this.config.node.dev - ) { - return; - } - - await using lock = await this.mutex.acquire('init stripe prices'); - - if (!lock) { - return; - } - - const keys = new Set(); - try { - await this.stripe.prices - .list({ - active: true, - limit: 100, - }) - .autoPagingEach(item => { - if (item.lookup_key) { - keys.add(item.lookup_key); - } - }); - } catch { - this.logger.warn('Failed to list stripe prices, skip auto init.'); - return; - } - - for (const [key, setting] of DEFAULT_PRICES) { - if (keys.has(key)) { - continue; - } - - const lookupKey = decodeLookupKey(key); - - try { - await this.stripe.prices.create({ - product_data: { - name: setting.product, - }, - billing_scheme: 'per_unit', - unit_amount: setting.price, - currency: 'usd', - lookup_key: key, - tax_behavior: 'inclusive', - recurring: - lookupKey.recurring === SubscriptionRecurring.Lifetime || - lookupKey.variant === SubscriptionVariant.Onetime - ? undefined - : { - interval: - lookupKey.recurring === SubscriptionRecurring.Monthly - ? 'month' - : 'year', - interval_count: 1, - usage_type: 'licensed', - }, - }); - } catch (e) { - this.logger.error('Failed to create stripe price.', e); - } - } - } } diff --git a/packages/backend/server/src/plugins/payment/stripe.ts b/packages/backend/server/src/plugins/payment/stripe.ts index 0ee8b8e43e..68ccc8a34c 100644 --- a/packages/backend/server/src/plugins/payment/stripe.ts +++ b/packages/backend/server/src/plugins/payment/stripe.ts @@ -1,18 +1,130 @@ -import assert from 'node:assert'; - -import { FactoryProvider } from '@nestjs/common'; -import { omit } from 'lodash-es'; +import { FactoryProvider, Injectable, Logger } from '@nestjs/common'; import Stripe from 'stripe'; -import { Config } from '../../base'; +import { Config, Mutex, OnEvent } from '../../base'; +import { ServerFeature, ServerService } from '../../core'; +import { + decodeLookupKey, + DEFAULT_PRICES, + SubscriptionRecurring, + SubscriptionVariant, +} from './types'; + +@Injectable() +export class StripeFactory { + #stripe!: Stripe; + readonly #logger = new Logger(StripeFactory.name); + + constructor( + private readonly config: Config, + private readonly mutex: Mutex, + private readonly server: ServerService + ) {} + + get stripe() { + return this.#stripe; + } + + @OnEvent('config.init') + async onConfigInit() { + this.setup(); + await this.initStripeProducts(); + } + + @OnEvent('config.changed') + async onConfigChanged(event: Events['config.changed']) { + if ('payment' in event.updates) { + this.setup(); + } + } + + setup() { + // TODO@(@forehalo): use per-requests api key injection + this.#stripe = new Stripe( + this.config.payment.apiKey || + // NOTE(@forehalo): + // we always fake a key if not set because `new Stripe` will complain if it's empty string + // this will make code cleaner than providing `Stripe` instance as optional one. + 'stripe-api-key', + this.config.payment.stripe + ); + if (this.config.payment.enabled) { + this.server.enableFeature(ServerFeature.Payment); + } else { + this.server.disableFeature(ServerFeature.Payment); + } + } + + private async initStripeProducts() { + // only init stripe products in dev mode or canary deployment + if (!this.config.payment.enabled && !env.namespaces.canary) { + return; + } + + await using lock = await this.mutex.acquire('init stripe prices'); + + if (!lock) { + return; + } + + const keys = new Set(); + try { + await this.stripe.prices + .list({ + active: true, + limit: 100, + }) + .autoPagingEach(item => { + if (item.lookup_key) { + keys.add(item.lookup_key); + } + }); + } catch { + this.#logger.warn('Failed to list stripe prices, skip auto init.'); + return; + } + + for (const [key, setting] of DEFAULT_PRICES) { + if (keys.has(key)) { + continue; + } + + const lookupKey = decodeLookupKey(key); + + try { + await this.stripe.prices.create({ + product_data: { + name: setting.product, + }, + billing_scheme: 'per_unit', + unit_amount: setting.price, + currency: 'usd', + lookup_key: key, + tax_behavior: 'inclusive', + recurring: + lookupKey.recurring === SubscriptionRecurring.Lifetime || + lookupKey.variant === SubscriptionVariant.Onetime + ? undefined + : { + interval: + lookupKey.recurring === SubscriptionRecurring.Monthly + ? 'month' + : 'year', + interval_count: 1, + usage_type: 'licensed', + }, + }); + } catch (e) { + this.#logger.error('Failed to create stripe price.', e); + } + } + } +} export const StripeProvider: FactoryProvider = { provide: Stripe, - useFactory: (config: Config) => { - const stripeConfig = config.plugins.payment.stripe; - assert(stripeConfig, 'Stripe configuration is missing'); - - return new Stripe(stripeConfig.keys.APIKey, omit(stripeConfig, 'keys')); + useFactory: (provider: StripeFactory) => { + return provider.stripe; }, - inject: [Config], + inject: [StripeFactory], }; diff --git a/packages/backend/server/src/plugins/payment/webhook.ts b/packages/backend/server/src/plugins/payment/webhook.ts index c295ea8646..67bfd553c1 100644 --- a/packages/backend/server/src/plugins/payment/webhook.ts +++ b/packages/backend/server/src/plugins/payment/webhook.ts @@ -3,6 +3,7 @@ import Stripe from 'stripe'; import { OnEvent } from '../../base'; import { SubscriptionService } from './service'; +import { StripeFactory } from './stripe'; /** * Stripe webhook events sent in random order, and may be even sent more than once. @@ -14,9 +15,13 @@ import { SubscriptionService } from './service'; export class StripeWebhook { constructor( private readonly service: SubscriptionService, - private readonly stripe: Stripe + private readonly stripeProvider: StripeFactory ) {} + get stripe() { + return this.stripeProvider.stripe; + } + @OnEvent('stripe.invoice.created') @OnEvent('stripe.invoice.updated') @OnEvent('stripe.invoice.finalization_failed') diff --git a/packages/backend/server/src/plugins/registry.ts b/packages/backend/server/src/plugins/registry.ts deleted file mode 100644 index 2d9dae9399..0000000000 --- a/packages/backend/server/src/plugins/registry.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { get, merge, omit, set } from 'lodash-es'; - -import { OptionalModule, OptionalModuleMetadata } from '../base/nestjs'; -import { AvailablePlugins } from './config'; - -export const REGISTERED_PLUGINS = new Map(); -export const ENABLED_PLUGINS = new Set(); - -function registerPlugin(plugin: AvailablePlugins, module: AFFiNEModule) { - REGISTERED_PLUGINS.set(plugin, module); -} - -interface PluginModuleMetadata extends OptionalModuleMetadata { - name: AvailablePlugins; -} - -export const Plugin = (options: PluginModuleMetadata) => { - return (target: any) => { - registerPlugin(options.name, target); - - return OptionalModule(omit(options, 'name'))(target); - }; -}; - -export function enablePlugin(plugin: AvailablePlugins, config: any = {}) { - config = merge(get(AFFiNE.plugins, plugin), config); - set(AFFiNE.plugins, plugin, config); - - ENABLED_PLUGINS.add(plugin); -} diff --git a/packages/backend/server/src/plugins/storage/config.ts b/packages/backend/server/src/plugins/storage/config.ts deleted file mode 100644 index 780d1ddf4a..0000000000 --- a/packages/backend/server/src/plugins/storage/config.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { S3ClientConfig, S3ClientConfigType } from '@aws-sdk/client-s3'; - -import { defineStartupConfig, ModuleConfig } from '../../base/config'; - -type WARNING = '__YOU_SHOULD_NOT_MANUALLY_CONFIGURATE_THIS_TYPE__'; -declare module '../../base/storage/config' { - interface StorageProvidersConfig { - // the type here is only existing for extends [StorageProviderType] with better type inference and checking. - 'cloudflare-r2'?: WARNING; - 'aws-s3'?: WARNING; - } -} - -export type S3StorageConfig = S3ClientConfigType; -export type R2StorageConfig = S3ClientConfigType & { - accountId?: string; -}; - -declare module '../config' { - interface PluginsConfig { - 'aws-s3': ModuleConfig; - 'cloudflare-r2': ModuleConfig; - } -} - -defineStartupConfig('plugins.aws-s3', {}); -defineStartupConfig('plugins.cloudflare-r2', {}); diff --git a/packages/backend/server/src/plugins/storage/index.ts b/packages/backend/server/src/plugins/storage/index.ts deleted file mode 100644 index d4b74f472f..0000000000 --- a/packages/backend/server/src/plugins/storage/index.ts +++ /dev/null @@ -1,42 +0,0 @@ -import './config'; - -import { registerStorageProvider } from '../../base/storage'; -import { Plugin } from '../registry'; -import { R2StorageProvider } from './providers/r2'; -import { S3StorageProvider } from './providers/s3'; - -registerStorageProvider('cloudflare-r2', (config, bucket) => { - if (!config.plugins['cloudflare-r2']) { - throw new Error('Missing cloudflare-r2 storage provider configuration'); - } - - return new R2StorageProvider(config.plugins['cloudflare-r2'], bucket); -}); -registerStorageProvider('aws-s3', (config, bucket) => { - if (!config.plugins['aws-s3']) { - throw new Error('Missing aws-s3 storage provider configuration'); - } - - return new S3StorageProvider(config.plugins['aws-s3'], bucket); -}); - -@Plugin({ - name: 'cloudflare-r2', - requires: [ - 'plugins.cloudflare-r2.accountId', - 'plugins.cloudflare-r2.credentials.accessKeyId', - 'plugins.cloudflare-r2.credentials.secretAccessKey', - ], - if: config => config.flavor.graphql, -}) -export class CloudflareR2Module {} - -@Plugin({ - name: 'aws-s3', - requires: [ - 'plugins.aws-s3.credentials.accessKeyId', - 'plugins.aws-s3.credentials.secretAccessKey', - ], - if: config => config.flavor.graphql, -}) -export class AwsS3Module {} diff --git a/packages/backend/server/src/plugins/worker/config.ts b/packages/backend/server/src/plugins/worker/config.ts index c37c07946a..9fdfebcc6c 100644 --- a/packages/backend/server/src/plugins/worker/config.ts +++ b/packages/backend/server/src/plugins/worker/config.ts @@ -1,15 +1,20 @@ -import { defineStartupConfig, ModuleConfig } from '../../base/config'; +import { defineModuleConfig } from '../../base'; export interface WorkerStartupConfigurations { allowedOrigin: string[]; } -declare module '../config' { - interface PluginsConfig { - worker: ModuleConfig; +declare global { + interface AppConfigSchema { + worker: { + allowedOrigin: ConfigItem; + }; } } -defineStartupConfig('plugins.worker', { - allowedOrigin: ['localhost', '127.0.0.1'], +defineModuleConfig('worker', { + allowedOrigin: { + desc: 'Allowed origin', + default: ['localhost', '127.0.0.1'], + }, }); diff --git a/packages/backend/server/src/plugins/worker/controller.ts b/packages/backend/server/src/plugins/worker/controller.ts index ef92cfec66..b299eb941d 100644 --- a/packages/backend/server/src/plugins/worker/controller.ts +++ b/packages/backend/server/src/plugins/worker/controller.ts @@ -10,8 +10,9 @@ import { import type { Request, Response } from 'express'; import { HTMLRewriter } from 'htmlrewriter'; -import { BadRequest, Cache, Config, URLHelper } from '../../base'; +import { BadRequest, Cache, URLHelper, UseNamedGuard } from '../../base'; import { Public } from '../../core/auth'; +import { WorkerService } from './service'; import type { LinkPreviewRequest, LinkPreviewResponse } from './types'; import { appendUrl, @@ -20,7 +21,6 @@ import { getCorsHeaders, isOriginAllowed, isRefererAllowed, - OriginRules, parseJson, reduceUrls, } from './utils'; @@ -30,22 +30,19 @@ import { decodeWithCharset } from './utils/encoding'; const CACHE_TTL = 1000 * 60 * 30; @Public() +@UseNamedGuard('selfhost') @Controller('/api/worker') export class WorkerController { private readonly logger = new Logger(WorkerController.name); - private readonly allowedOrigin: OriginRules; constructor( - config: Config, private readonly cache: Cache, - private readonly url: URLHelper - ) { - this.allowedOrigin = [ - ...config.plugins.worker.allowedOrigin - .map(u => fixUrl(u)?.origin as string) - .filter(v => !!v), - url.origin, - ]; + private readonly url: URLHelper, + private readonly service: WorkerService + ) {} + + private get allowedOrigin() { + return this.service.allowedOrigins; } @Get('/image-proxy') diff --git a/packages/backend/server/src/plugins/worker/index.ts b/packages/backend/server/src/plugins/worker/index.ts index 1878539675..a90fbdc494 100644 --- a/packages/backend/server/src/plugins/worker/index.ts +++ b/packages/backend/server/src/plugins/worker/index.ts @@ -1,11 +1,11 @@ import './config'; -import { Plugin } from '../registry'; -import { WorkerController } from './controller'; +import { Module } from '@nestjs/common'; -@Plugin({ - name: 'worker', +import { WorkerController } from './controller'; +import { WorkerService } from './service'; +@Module({ + providers: [WorkerService], controllers: [WorkerController], - if: config => config.isSelfhosted || config.node.dev || config.node.test, }) export class WorkerModule {} diff --git a/packages/backend/server/src/plugins/worker/service.ts b/packages/backend/server/src/plugins/worker/service.ts new file mode 100644 index 0000000000..4bf0a8d602 --- /dev/null +++ b/packages/backend/server/src/plugins/worker/service.ts @@ -0,0 +1,31 @@ +import { Injectable } from '@nestjs/common'; + +import { Config, OnEvent, URLHelper } from '../../base'; +import { fixUrl, OriginRules } from './utils'; + +@Injectable() +export class WorkerService { + allowedOrigins: OriginRules = [this.url.origin]; + + constructor( + private readonly config: Config, + private readonly url: URLHelper + ) {} + + @OnEvent('config.init') + onConfigInit() { + this.allowedOrigins = [ + ...this.config.worker.allowedOrigin + .map(u => fixUrl(u)?.origin as string) + .filter(v => !!v), + this.url.origin, + ]; + } + + @OnEvent('config.changed') + onConfigChanged(event: Events['config.changed']) { + if ('worker' in event.updates) { + this.onConfigInit(); + } + } +} diff --git a/packages/backend/server/src/prelude.ts b/packages/backend/server/src/prelude.ts index 20c2c8345d..469da6c696 100644 --- a/packages/backend/server/src/prelude.ts +++ b/packages/backend/server/src/prelude.ts @@ -1,41 +1,14 @@ import 'reflect-metadata'; +import './env'; -import { cpSync, existsSync, readFileSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { homedir } from 'node:os'; -import { join, parse } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; +import { join } from 'node:path'; import { config } from 'dotenv'; -import { applyEnvToConfig, getAFFiNEConfigModifier } from './base/config'; - -const PROJECT_CONFIG_PATH = join(fileURLToPath(import.meta.url), '../config'); const CUSTOM_CONFIG_PATH = `${homedir()}/.affine/config`; -async function loadConfig(configDir: string, file: string) { - let fileToLoad: string | undefined; - - if (PROJECT_CONFIG_PATH !== configDir) { - const remoteFile = join(configDir, file); - const remoteFileAtLocal = join( - PROJECT_CONFIG_PATH, - parse(file).name + '.remote.js' - ); - if (existsSync(remoteFile)) { - cpSync(remoteFile, remoteFileAtLocal, { - force: true, - }); - fileToLoad = remoteFileAtLocal; - } - } else { - fileToLoad = join(PROJECT_CONFIG_PATH, file); - } - - if (fileToLoad) { - await import(pathToFileURL(fileToLoad).href); - } -} - function loadPrivateKey() { const file = join(CUSTOM_CONFIG_PATH, 'private.key'); if (!process.env.AFFINE_PRIVATE_KEY && existsSync(file)) { @@ -44,53 +17,23 @@ function loadPrivateKey() { } } -async function load() { +function load() { let isPrivateKeyFromEnv = !!process.env.AFFINE_PRIVATE_KEY; - // Initializing AFFiNE config - // - // 1. load dotenv file to `process.env` // load `.env` under pwd config(); - // @deprecated removed // load `.env` under user config folder config({ path: join(CUSTOM_CONFIG_PATH, '.env'), }); - // @deprecated - // The old AFFINE_PRIVATE_KEY in old .env is somehow not working, we should ignore it + // The old AFFINE_PRIVATE_KEY in old .env is somehow not working, + // we should ignore it if (!isPrivateKeyFromEnv) { delete process.env.AFFINE_PRIVATE_KEY; } - // 2. generate AFFiNE default config and assign to `globalThis.AFFiNE` - globalThis.AFFiNE = getAFFiNEConfigModifier(); - const { enablePlugin } = await import('./plugins/registry'); - globalThis.AFFiNE.use = enablePlugin; - globalThis.AFFiNE.plugins.use = enablePlugin; - - // TODO(@forehalo): - // Modules may contribute to ENV_MAP, figure out a good way to involve them instead of hardcoding in `./config/affine.env` - // 3. load env => config map to `globalThis.AFFiNE.ENV_MAP - // load local env map as well in case there are new env added - await loadConfig(PROJECT_CONFIG_PATH, 'affine.env.js'); - - // 4. load `config/affine` to patch custom configs - // load local config as well in case there are new default configurations added - await loadConfig(PROJECT_CONFIG_PATH, 'affine.js'); - await loadConfig(CUSTOM_CONFIG_PATH, 'affine.js'); - - // 5. load `config/affine.self` to patch custom configs - // This is the file only take effect in [AFFiNE Cloud] - if (!AFFiNE.isSelfhosted) { - await loadConfig(PROJECT_CONFIG_PATH, 'affine.self.js'); - } - - // 6. load `config/private.key` to patch app configs + // 2. load `config/private.key` to patch app configs loadPrivateKey(); - - // 7. apply `process.env` map overriding to `globalThis.AFFiNE` - applyEnvToConfig(globalThis.AFFiNE); } -await load(); +load(); diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 37feeadb8b..81c31aa70f 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -458,6 +458,7 @@ enum ErrorNames { GRAPHQL_BAD_REQUEST HTTP_REQUEST_ERROR INTERNAL_SERVER_ERROR + INVALID_APP_CONFIG INVALID_AUTH_STATE INVALID_CHECKOUT_PARAMETERS INVALID_EMAIL @@ -1002,6 +1003,9 @@ type Mutation { setBlob(blob: Upload!, workspaceId: String!): String! submitAudioTranscription(blob: Upload!, blobId: String!, workspaceId: String!): TranscriptionResultType + """update app configuration""" + updateAppConfig(updates: [UpdateAppConfigInput!]!): JSONObject! + """Update a copilot prompt""" updateCopilotPrompt(messages: [CopilotPromptMessageInput!]!, name: String!): CopilotPromptType! @@ -1011,12 +1015,6 @@ type Mutation { updateDocUserRole(input: UpdateDocUserRoleInput!): Boolean! updateProfile(input: UpdateUserInput!): UserType! - """update server runtime configurable setting""" - updateRuntimeConfig(id: String!, value: JSON!): ServerRuntimeConfigType! - - """update multiple server runtime configurable settings""" - updateRuntimeConfigs(updates: JSONObject!): [ServerRuntimeConfigType!]! - """Update user settings""" updateSettings(input: UpdateUserSettingsInput!): Boolean! updateSubscriptionRecurring(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, recurring: SubscriptionRecurring!, workspaceId: String): SubscriptionType! @@ -1162,6 +1160,8 @@ type PublicUserType { } type Query { + """get the whole app configuration""" + appConfig: JSONObject! collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.quotaUsage` instead") """Get current user""" @@ -1190,10 +1190,6 @@ type Query { """server config""" serverConfig: ServerConfigType! - """get all server runtime configurable settings""" - serverRuntimeConfig: [ServerRuntimeConfigType!]! - serverServiceConfigs: [ServerServiceConfig!]! - """Get user by email""" user(email: String!): UserOrLimitedUser @@ -1275,14 +1271,6 @@ type RuntimeConfigNotFoundDataType { key: String! } -enum RuntimeConfigType { - Array - Boolean - Number - Object - String -} - """ The `SafeInt` scalar type represents non-fractional signed whole numeric values that are considered safe as defined by the ECMAScript specification. """ @@ -1294,7 +1282,7 @@ type SameSubscriptionRecurringDataType { type ServerConfigType { """fetch latest available upgradable release of server""" - availableUpgrade: ReleaseVersionType! + availableUpgrade: ReleaseVersionType """Features for user that can be configured""" availableUserFeatures: [FeatureType!]! @@ -1305,18 +1293,9 @@ type ServerConfigType { """credentials requirement""" credentialsRequirement: CredentialsRequirementType! - """enable telemetry""" - enableTelemetry: Boolean! - """enabled server features""" features: [ServerFeature!]! - """server flags""" - flags: ServerFlagsType! - - """server flavor""" - flavor: String! @deprecated(reason: "use `features`") - """whether server has been initialized""" initialized: Boolean! @@ -1343,26 +1322,6 @@ enum ServerFeature { Payment } -type ServerFlagsType { - earlyAccessControl: Boolean! - syncClientVersionCheck: Boolean! -} - -type ServerRuntimeConfigType { - description: String! - id: String! - key: String! - module: String! - type: RuntimeConfigType! - updatedAt: DateTime! - value: JSON! -} - -type ServerServiceConfig { - config: JSONObject! - name: String! -} - type SpaceAccessDeniedDataType { spaceId: String! } @@ -1484,6 +1443,12 @@ type UnsupportedSubscriptionPlanDataType { plan: String! } +input UpdateAppConfigInput { + key: String! + module: String! + value: JSON! +} + input UpdateChatSessionInput { """The prompt name to use for the session""" promptName: String! diff --git a/packages/common/graphql/src/graphql/admin/config.gql b/packages/common/graphql/src/graphql/admin/config.gql new file mode 100644 index 0000000000..017ccb83e1 --- /dev/null +++ b/packages/common/graphql/src/graphql/admin/config.gql @@ -0,0 +1,3 @@ +query appConfig { + appConfig +} \ No newline at end of file diff --git a/packages/common/graphql/src/graphql/admin/get-server-runtime-config.gql b/packages/common/graphql/src/graphql/admin/get-server-runtime-config.gql deleted file mode 100644 index 9229b3c7e8..0000000000 --- a/packages/common/graphql/src/graphql/admin/get-server-runtime-config.gql +++ /dev/null @@ -1,11 +0,0 @@ -query getServerRuntimeConfig { - serverRuntimeConfig { - id - module - key - description - value - type - updatedAt - } -} diff --git a/packages/common/graphql/src/graphql/admin/get-server-service-configs.gql b/packages/common/graphql/src/graphql/admin/get-server-service-configs.gql deleted file mode 100644 index 279e77d15a..0000000000 --- a/packages/common/graphql/src/graphql/admin/get-server-service-configs.gql +++ /dev/null @@ -1,6 +0,0 @@ -query getServerServiceConfigs { - serverServiceConfigs { - name - config - } -} diff --git a/packages/common/graphql/src/graphql/admin/update-config.gql b/packages/common/graphql/src/graphql/admin/update-config.gql new file mode 100644 index 0000000000..25bfd10ec4 --- /dev/null +++ b/packages/common/graphql/src/graphql/admin/update-config.gql @@ -0,0 +1,3 @@ +mutation updateAppConfig($updates: [UpdateAppConfigInput!]!) { + updateAppConfig(updates: $updates) +} \ No newline at end of file diff --git a/packages/common/graphql/src/graphql/admin/update-server-runtime-configs.gql b/packages/common/graphql/src/graphql/admin/update-server-runtime-configs.gql deleted file mode 100644 index 0ab081950c..0000000000 --- a/packages/common/graphql/src/graphql/admin/update-server-runtime-configs.gql +++ /dev/null @@ -1,6 +0,0 @@ -mutation updateServerRuntimeConfigs($updates: JSONObject!) { - updateRuntimeConfigs(updates: $updates) { - key - value - } -} diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index b6bb173e78..b72484147b 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -61,6 +61,14 @@ export const createChangePasswordUrlMutation = { }`, }; +export const appConfigQuery = { + id: 'appConfigQuery' as const, + op: 'appConfig', + query: `query appConfig { + appConfig +}`, +}; + export const getPromptsQuery = { id: 'getPromptsQuery' as const, op: 'getPrompts', @@ -151,33 +159,6 @@ export const enableUserMutation = { }`, }; -export const getServerRuntimeConfigQuery = { - id: 'getServerRuntimeConfigQuery' as const, - op: 'getServerRuntimeConfig', - query: `query getServerRuntimeConfig { - serverRuntimeConfig { - id - module - key - description - value - type - updatedAt - } -}`, -}; - -export const getServerServiceConfigsQuery = { - id: 'getServerServiceConfigsQuery' as const, - op: 'getServerServiceConfigs', - query: `query getServerServiceConfigs { - serverServiceConfigs { - name - config - } -}`, -}; - export const getUserByEmailQuery = { id: 'getUserByEmailQuery' as const, op: 'getUserByEmail', @@ -259,14 +240,11 @@ export const updateAccountMutation = { }`, }; -export const updateServerRuntimeConfigsMutation = { - id: 'updateServerRuntimeConfigsMutation' as const, - op: 'updateServerRuntimeConfigs', - query: `mutation updateServerRuntimeConfigs($updates: JSONObject!) { - updateRuntimeConfigs(updates: $updates) { - key - value - } +export const updateAppConfigMutation = { + id: 'updateAppConfigMutation' as const, + op: 'updateAppConfig', + query: `mutation updateAppConfig($updates: [UpdateAppConfigInput!]!) { + updateAppConfig(updates: $updates) }`, }; diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index 9243173d6a..d2f8d2be80 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -603,6 +603,7 @@ export enum ErrorNames { GRAPHQL_BAD_REQUEST = 'GRAPHQL_BAD_REQUEST', HTTP_REQUEST_ERROR = 'HTTP_REQUEST_ERROR', INTERNAL_SERVER_ERROR = 'INTERNAL_SERVER_ERROR', + INVALID_APP_CONFIG = 'INVALID_APP_CONFIG', INVALID_AUTH_STATE = 'INVALID_AUTH_STATE', INVALID_CHECKOUT_PARAMETERS = 'INVALID_CHECKOUT_PARAMETERS', INVALID_EMAIL = 'INVALID_EMAIL', @@ -1114,6 +1115,8 @@ export interface Mutation { sendVerifyEmail: Scalars['Boolean']['output']; setBlob: Scalars['String']['output']; submitAudioTranscription: Maybe; + /** update app configuration */ + updateAppConfig: Scalars['JSONObject']['output']; /** Update a copilot prompt */ updateCopilotPrompt: CopilotPromptType; /** Update a chat session */ @@ -1121,10 +1124,6 @@ export interface Mutation { updateDocDefaultRole: Scalars['Boolean']['output']; updateDocUserRole: Scalars['Boolean']['output']; updateProfile: UserType; - /** update server runtime configurable setting */ - updateRuntimeConfig: ServerRuntimeConfigType; - /** update multiple server runtime configurable settings */ - updateRuntimeConfigs: Array; /** Update user settings */ updateSettings: Scalars['Boolean']['output']; updateSubscriptionRecurring: SubscriptionType; @@ -1426,6 +1425,10 @@ export interface MutationSubmitAudioTranscriptionArgs { workspaceId: Scalars['String']['input']; } +export interface MutationUpdateAppConfigArgs { + updates: Array; +} + export interface MutationUpdateCopilotPromptArgs { messages: Array; name: Scalars['String']['input']; @@ -1447,15 +1450,6 @@ export interface MutationUpdateProfileArgs { input: UpdateUserInput; } -export interface MutationUpdateRuntimeConfigArgs { - id: Scalars['String']['input']; - value: Scalars['JSON']['input']; -} - -export interface MutationUpdateRuntimeConfigsArgs { - updates: Scalars['JSONObject']['input']; -} - export interface MutationUpdateSettingsArgs { input: UpdateUserSettingsInput; } @@ -1615,6 +1609,8 @@ export interface PublicUserType { export interface Query { __typename?: 'Query'; + /** get the whole app configuration */ + appConfig: Scalars['JSONObject']['output']; /** @deprecated use `user.quotaUsage` instead */ collectAllBlobSizes: WorkspaceBlobSizes; /** Get current user */ @@ -1641,9 +1637,6 @@ export interface Query { queryWorkspaceEmbeddingStatus: ContextWorkspaceEmbeddingStatus; /** server config */ serverConfig: ServerConfigType; - /** get all server runtime configurable settings */ - serverRuntimeConfig: Array; - serverServiceConfigs: Array; /** Get user by email */ user: Maybe; /** Get user by email for admin */ @@ -1773,14 +1766,6 @@ export interface RuntimeConfigNotFoundDataType { key: Scalars['String']['output']; } -export enum RuntimeConfigType { - Array = 'Array', - Boolean = 'Boolean', - Number = 'Number', - Object = 'Object', - String = 'String', -} - export interface SameSubscriptionRecurringDataType { __typename?: 'SameSubscriptionRecurringDataType'; recurring: Scalars['String']['output']; @@ -1789,24 +1774,15 @@ export interface SameSubscriptionRecurringDataType { export interface ServerConfigType { __typename?: 'ServerConfigType'; /** fetch latest available upgradable release of server */ - availableUpgrade: ReleaseVersionType; + availableUpgrade: Maybe; /** Features for user that can be configured */ availableUserFeatures: Array; /** server base url */ baseUrl: Scalars['String']['output']; /** credentials requirement */ credentialsRequirement: CredentialsRequirementType; - /** enable telemetry */ - enableTelemetry: Scalars['Boolean']['output']; /** enabled server features */ features: Array; - /** server flags */ - flags: ServerFlagsType; - /** - * server flavor - * @deprecated use `features` - */ - flavor: Scalars['String']['output']; /** whether server has been initialized */ initialized: Scalars['Boolean']['output']; /** server identical name could be shown as badge on user interface */ @@ -1830,29 +1806,6 @@ export enum ServerFeature { Payment = 'Payment', } -export interface ServerFlagsType { - __typename?: 'ServerFlagsType'; - earlyAccessControl: Scalars['Boolean']['output']; - syncClientVersionCheck: Scalars['Boolean']['output']; -} - -export interface ServerRuntimeConfigType { - __typename?: 'ServerRuntimeConfigType'; - description: Scalars['String']['output']; - id: Scalars['String']['output']; - key: Scalars['String']['output']; - module: Scalars['String']['output']; - type: RuntimeConfigType; - updatedAt: Scalars['DateTime']['output']; - value: Scalars['JSON']['output']; -} - -export interface ServerServiceConfig { - __typename?: 'ServerServiceConfig'; - config: Scalars['JSONObject']['output']; - name: Scalars['String']['output']; -} - export interface SpaceAccessDeniedDataType { __typename?: 'SpaceAccessDeniedDataType'; spaceId: Scalars['String']['output']; @@ -1995,6 +1948,12 @@ export interface UnsupportedSubscriptionPlanDataType { plan: Scalars['String']['output']; } +export interface UpdateAppConfigInput { + key: Scalars['String']['input']; + module: Scalars['String']['input']; + value: Scalars['JSON']['input']; +} + export interface UpdateChatSessionInput { /** The prompt name to use for the session */ promptName: Scalars['String']['input']; @@ -2387,7 +2346,7 @@ export type AdminServerConfigQuery = { version: string; publishedAt: string; url: string; - }; + } | null; }; }; @@ -2401,6 +2360,10 @@ export type CreateChangePasswordUrlMutation = { createChangePasswordUrl: string; }; +export type AppConfigQueryVariables = Exact<{ [key: string]: never }>; + +export type AppConfigQuery = { __typename?: 'Query'; appConfig: any }; + export type GetPromptsQueryVariables = Exact<{ [key: string]: never }>; export type GetPromptsQuery = { @@ -2492,37 +2455,6 @@ export type EnableUserMutation = { enableUser: { __typename?: 'UserType'; email: string; disabled: boolean }; }; -export type GetServerRuntimeConfigQueryVariables = Exact<{ - [key: string]: never; -}>; - -export type GetServerRuntimeConfigQuery = { - __typename?: 'Query'; - serverRuntimeConfig: Array<{ - __typename?: 'ServerRuntimeConfigType'; - id: string; - module: string; - key: string; - description: string; - value: Record; - type: RuntimeConfigType; - updatedAt: string; - }>; -}; - -export type GetServerServiceConfigsQueryVariables = Exact<{ - [key: string]: never; -}>; - -export type GetServerServiceConfigsQuery = { - __typename?: 'Query'; - serverServiceConfigs: Array<{ - __typename?: 'ServerServiceConfig'; - name: string; - config: any; - }>; -}; - export type GetUserByEmailQueryVariables = Exact<{ email: Scalars['String']['input']; }>; @@ -2602,17 +2534,13 @@ export type UpdateAccountMutation = { }; }; -export type UpdateServerRuntimeConfigsMutationVariables = Exact<{ - updates: Scalars['JSONObject']['input']; +export type UpdateAppConfigMutationVariables = Exact<{ + updates: Array | UpdateAppConfigInput; }>; -export type UpdateServerRuntimeConfigsMutation = { +export type UpdateAppConfigMutation = { __typename?: 'Mutation'; - updateRuntimeConfigs: Array<{ - __typename?: 'ServerRuntimeConfigType'; - key: string; - value: Record; - }>; + updateAppConfig: any; }; export type DeleteBlobMutationVariables = Exact<{ @@ -4347,21 +4275,16 @@ export type Queries = variables: AdminServerConfigQueryVariables; response: AdminServerConfigQuery; } + | { + name: 'appConfigQuery'; + variables: AppConfigQueryVariables; + response: AppConfigQuery; + } | { name: 'getPromptsQuery'; variables: GetPromptsQueryVariables; response: GetPromptsQuery; } - | { - name: 'getServerRuntimeConfigQuery'; - variables: GetServerRuntimeConfigQueryVariables; - response: GetServerRuntimeConfigQuery; - } - | { - name: 'getServerServiceConfigsQuery'; - variables: GetServerServiceConfigsQueryVariables; - response: GetServerServiceConfigsQuery; - } | { name: 'getUserByEmailQuery'; variables: GetUserByEmailQueryVariables; @@ -4675,9 +4598,9 @@ export type Mutations = response: UpdateAccountMutation; } | { - name: 'updateServerRuntimeConfigsMutation'; - variables: UpdateServerRuntimeConfigsMutationVariables; - response: UpdateServerRuntimeConfigsMutation; + name: 'updateAppConfigMutation'; + variables: UpdateAppConfigMutationVariables; + response: UpdateAppConfigMutation; } | { name: 'deleteBlobMutation'; diff --git a/packages/frontend/admin/package.json b/packages/frontend/admin/package.json index c905ce2060..999d55316d 100644 --- a/packages/frontend/admin/package.json +++ b/packages/frontend/admin/package.json @@ -5,6 +5,7 @@ "dependencies": { "@affine/component": "workspace:*", "@affine/core": "workspace:*", + "@affine/error": "workspace:*", "@affine/graphql": "workspace:*", "@blocksuite/icons": "^2.2.8", "@radix-ui/react-accordion": "^1.2.2", @@ -41,6 +42,7 @@ "cmdk": "^1.0.4", "embla-carousel-react": "^8.5.1", "input-otp": "^1.4.1", + "lodash-es": "^4.17.21", "lucide-react": "^0.484.0", "next-themes": "^0.4.4", "react": "^19.0.0", @@ -55,6 +57,7 @@ "zod": "^3.24.1" }, "devDependencies": { + "@types/lodash-es": "^4.17.12", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cross-env": "^7.0.3", diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json new file mode 100644 index 0000000000..3b732be37d --- /dev/null +++ b/packages/frontend/admin/src/config.json @@ -0,0 +1,350 @@ +{ + "redis": { + "db": { + "type": "Number", + "desc": "The database index of redis server to be used(Must be less than 10).", + "env": "REDIS_DATABASE" + }, + "host": { + "type": "String", + "desc": "The host of the redis server.", + "env": "REDIS_HOST" + }, + "port": { + "type": "Number", + "desc": "The port of the redis server.", + "env": "REDIS_PORT" + }, + "username": { + "type": "String", + "desc": "The username of the redis server.", + "env": "REDIS_USERNAME" + }, + "password": { + "type": "String", + "desc": "The password of the redis server.", + "env": "REDIS_PASSWORD" + }, + "ioredis": { + "type": "Object", + "desc": "The config for the ioredis client.", + "link": "https://github.com/luin/ioredis" + } + }, + "metrics": { + "enabled": { + "type": "Boolean", + "desc": "Enable metric and tracing collection" + } + }, + "graphql": { + "apolloDriverConfig": { + "type": "Object", + "desc": "The config for underlying nestjs GraphQL and apollo driver engine.", + "link": "https://docs.nestjs.com/graphql/quick-start" + } + }, + "crypto": { + "privateKey": { + "type": "String", + "desc": "The private key for used by the crypto module to create signed tokens or encrypt data.", + "env": "AFFINE_PRIVATE_KEY" + } + }, + "job": { + "queue": { + "type": "Object", + "desc": "The config for job queues", + "link": "https://api.docs.bullmq.io/interfaces/v5.QueueOptions.html" + }, + "worker": { + "type": "Object", + "desc": "The config for job workers", + "link": "https://api.docs.bullmq.io/interfaces/v5.WorkerOptions.html" + }, + "queues.copilot": { + "type": "Object", + "desc": "The config for copilot job queue" + }, + "queues.doc": { + "type": "Object", + "desc": "The config for doc job queue" + }, + "queues.notification": { + "type": "Object", + "desc": "The config for notification job queue" + }, + "queues.nightly": { + "type": "Object", + "desc": "The config for nightly job queue" + } + }, + "throttle": { + "enabled": { + "type": "Boolean", + "desc": "Whether the throttler is enabled." + }, + "throttlers.default": { + "type": "Object", + "desc": "The config for the default throttler." + }, + "throttlers.strict": { + "type": "Object", + "desc": "The config for the strict throttler." + } + }, + "websocket": { + "transports": { + "type": "Array", + "desc": "The enabled transports for accepting websocket traffics.", + "link": "https://docs.nestjs.com/websockets/gateways#transports" + }, + "maxHttpBufferSize": { + "type": "Number", + "desc": "How many bytes or characters a message can be, before closing the session (to avoid DoS)." + } + }, + "db": { + "datasourceUrl": { + "type": "String", + "desc": "The datasource url for the prisma client.", + "env": "DATABASE_URL" + }, + "prisma": { + "type": "Object", + "desc": "The config for the prisma client.", + "link": "https://www.prisma.io/docs/reference/api-reference/prisma-client-reference" + } + }, + "auth": { + "allowSignup": { + "type": "Boolean", + "desc": "Whether allow new registrations." + }, + "requireEmailDomainVerification": { + "type": "Boolean", + "desc": "Whether require email domain record verification before accessing restricted resources." + }, + "requireEmailVerification": { + "type": "Boolean", + "desc": "Whether require email verification before accessing restricted resources(not implemented)." + }, + "passwordRequirements": { + "type": "Object", + "desc": "The password strength requirements when set new password." + }, + "session.ttl": { + "type": "Number", + "desc": "Application auth expiration time in seconds." + }, + "session.ttr": { + "type": "Number", + "desc": "Application auth time to refresh in seconds." + } + }, + "mailer": { + "enabled": { + "type": "Boolean", + "desc": "Whether enabled mail service." + }, + "SMTP.host": { + "type": "String", + "desc": "Host of the email server (e.g. smtp.gmail.com)", + "env": "MAILER_HOST" + }, + "SMTP.port": { + "type": "Number", + "desc": "Port of the email server (they commonly are 25, 465 or 587)", + "env": "MAILER_PORT" + }, + "SMTP.username": { + "type": "String", + "desc": "Username used to authenticate the email server", + "env": "MAILER_USER" + }, + "SMTP.password": { + "type": "String", + "desc": "Password used to authenticate the email server", + "env": "MAILER_PASSWORD" + }, + "SMTP.sender": { + "type": "String", + "desc": "Sender of all the emails (e.g. \"AFFiNE Team \")", + "env": "MAILER_SENDER" + }, + "SMTP.ignoreTLS": { + "type": "Boolean", + "desc": "Whether ignore email server's TSL certification verification. Enable it for self-signed certificates.", + "env": "MAILER_IGNORE_TLS" + } + }, + "doc": { + "experimental.yocto": { + "type": "Boolean", + "desc": "Use `y-octo` to merge updates at the same time when merging using Yjs." + }, + "history.interval": { + "type": "Number", + "desc": "The minimum time interval in milliseconds of creating a new history snapshot when doc get updated." + } + }, + "storages": { + "avatar.publicPath": { + "type": "String", + "desc": "The public accessible path prefix for user avatars." + }, + "avatar.storage": { + "type": "Object", + "desc": "The config of storage for user avatars." + }, + "blob.storage": { + "type": "Object", + "desc": "The config of storage for all uploaded blobs(images, videos, etc.)." + } + }, + "server": { + "name": { + "type": "String", + "desc": "A recognizable name for the server. Will be shown when connected with AFFiNE Desktop." + }, + "externalUrl": { + "type": "String", + "desc": "Base url of AFFiNE server, used for generating external urls.\nDefault to be `[server.protocol]://[server.host][:server.port]` if not specified.\n ", + "env": "AFFINE_SERVER_EXTERNAL_URL" + }, + "https": { + "type": "Boolean", + "desc": "Whether the server is hosted on a ssl enabled domain (https://).", + "env": "AFFINE_SERVER_HTTPS" + }, + "host": { + "type": "String", + "desc": "Where the server get deployed(FQDN).", + "env": "AFFINE_SERVER_HOST" + }, + "port": { + "type": "Number", + "desc": "Which port the server will listen on.", + "env": "AFFINE_SERVER_PORT" + }, + "path": { + "type": "String", + "desc": "Subpath where the server get deployed if there is.", + "env": "AFFINE_SERVER_SUB_PATH" + } + }, + "flags": { + "earlyAccessControl": { + "type": "Boolean", + "desc": "Only allow users with early access features to access the app" + } + }, + "client": { + "versionControl.enabled": { + "type": "Boolean", + "desc": "Whether check version of client before accessing the server." + }, + "versionControl.requiredVersion": { + "type": "String", + "desc": "Allowed version range of the app that allowed to access the server. Requires 'client/versionControl.enabled' to be true to take effect." + } + }, + "captcha": { + "enabled": { + "type": "Boolean", + "desc": "Check captcha challenge when user authenticating the app." + }, + "config": { + "type": "Object", + "desc": "The config for the captcha plugin." + } + }, + "copilot": { + "enabled": { + "type": "Boolean", + "desc": "Whether to enable the copilot plugin." + }, + "providers.openai": { + "type": "Object", + "desc": "The config for the openai provider.", + "link": "https://github.com/openai/openai-node" + }, + "providers.fal": { + "type": "Object", + "desc": "The config for the fal provider." + }, + "providers.gemini": { + "type": "Object", + "desc": "The config for the gemini provider." + }, + "providers.perplexity": { + "type": "Object", + "desc": "The config for the perplexity provider." + }, + "unsplash": { + "type": "Object", + "desc": "The config for the unsplash key." + }, + "storage": { + "type": "Object", + "desc": "The config for the storage provider." + } + }, + "customerIo": { + "enabled": { + "type": "Boolean", + "desc": "Enable customer.io integration" + }, + "token": { + "type": "String", + "desc": "Customer.io token" + } + }, + "oauth": { + "providers.google": { + "type": "Object", + "desc": "Google OAuth provider config", + "link": "https://developers.google.com/identity/protocols/oauth2/web-server" + }, + "providers.github": { + "type": "Object", + "desc": "GitHub OAuth provider config", + "link": "https://docs.github.com/en/apps/oauth-apps" + }, + "providers.oidc": { + "type": "Object", + "desc": "OIDC OAuth provider config" + } + }, + "payment": { + "enabled": { + "type": "Boolean", + "desc": "Whether enable payment plugin" + }, + "showLifetimePrice": { + "type": "Boolean", + "desc": "Whether enable lifetime price and allow user to pay for it." + }, + "apiKey": { + "type": "String", + "desc": "Stripe API key to enable payment service.", + "env": "STRIPE_API_KEY" + }, + "webhookKey": { + "type": "String", + "desc": "Stripe webhook key to enable payment service.", + "env": "STRIPE_WEBHOOK_KEY" + }, + "stripe": { + "type": "Object", + "desc": "Stripe API keys", + "link": "https://docs.stripe.com/api" + } + }, + "worker": { + "allowedOrigin": { + "type": "Array", + "desc": "Allowed origin" + } + } +} \ No newline at end of file diff --git a/packages/frontend/admin/src/modules/config/index.tsx b/packages/frontend/admin/src/modules/config/index.tsx index 21a8b1a563..499527723a 100644 --- a/packages/frontend/admin/src/modules/config/index.tsx +++ b/packages/frontend/admin/src/modules/config/index.tsx @@ -1,177 +1,17 @@ -import { - Card, - CardContent, - CardHeader, - CardTitle, -} from '@affine/admin/components/ui/card'; import { ScrollArea } from '@affine/admin/components/ui/scroll-area'; import { Header } from '../header'; import { AboutAFFiNE } from './about'; -import type { - DatabaseConfig, - MailerConfig, - ServerConfig, -} from './use-server-service-configs'; -import { useServerServiceConfigs } from './use-server-service-configs'; export function ConfigPage() { return (

-
); } -const ServerCard = ({ serverConfig }: { serverConfig?: ServerConfig }) => { - if (!serverConfig) return null; - return ( - - - Server - - -
-
-
Domain
-
- {serverConfig.host} -
-
-
-
Port
-
- {serverConfig.port} -
-
-
-
HTTPS Prefix
-
- {serverConfig.https.toString()} -
-
-
-
External Url
-
- {serverConfig.externalUrl} -
-
-
-
-
- ); -}; -const DatabaseCard = ({ - databaseConfig, -}: { - databaseConfig?: DatabaseConfig; -}) => { - if (!databaseConfig) return null; - return ( - - - Database - - -
-
-
Domain
-
- {databaseConfig.host} -
-
-
-
Port
-
- {databaseConfig.port} -
-
-
-
User
-
- {databaseConfig.user} -
-
-
-
Database
-
- {databaseConfig.database} -
-
-
-
-
- ); -}; -const MailerCard = ({ mailerConfig }: { mailerConfig?: MailerConfig }) => { - if (!mailerConfig) return null; - return ( - - - Email - - -
-
-
Provider Domain
-
- {mailerConfig.host} -
-
-
-
Port
-
- {mailerConfig.port} -
-
-
-
Sender
-
- {mailerConfig.sender} -
-
-
-
-
- ); -}; - -export function ServerServiceConfig() { - const { serverConfig, mailerConfig, databaseConfig } = - useServerServiceConfigs(); - - return ( -
-
- Server Config -
-
-
- - -
-
- -
- - These settings are controlled by Docker environment variables. - Refer to the - - - Selfhost documentation. - -
-
-
-
- ); -} - export { ConfigPage as Component }; diff --git a/packages/frontend/admin/src/modules/config/use-server-service-configs.ts b/packages/frontend/admin/src/modules/config/use-server-service-configs.ts deleted file mode 100644 index bcd439ca0f..0000000000 --- a/packages/frontend/admin/src/modules/config/use-server-service-configs.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { useQueryImmutable } from '@affine/admin/use-query'; -import { getServerServiceConfigsQuery } from '@affine/graphql'; -import { useMemo } from 'react'; - -export type ServerConfig = { - externalUrl: string; - https: boolean; - host: string; - port: number; - path: string; -}; - -export type MailerConfig = { - host: string; - port: number; - sender: string; -}; - -export type DatabaseConfig = { - host: string; - port: number; - user: string; - database: string; -}; - -export type ServerServiceConfig = { - name: string; - config: ServerConfig | MailerConfig | DatabaseConfig; -}; - -export const useServerServiceConfigs = () => { - const { data } = useQueryImmutable({ - query: getServerServiceConfigsQuery, - }); - const server = useMemo( - () => - data.serverServiceConfigs.find( - (service: ServerServiceConfig) => service.name === 'server' - ), - [data.serverServiceConfigs] - ); - const mailer = useMemo( - () => - data.serverServiceConfigs.find( - (service: ServerServiceConfig) => service.name === 'mailer' - ), - [data.serverServiceConfigs] - ); - const database = useMemo( - () => - data.serverServiceConfigs.find( - (service: ServerServiceConfig) => service.name === 'database' - ), - [data.serverServiceConfigs] - ); - - const serverConfig = server?.config as ServerConfig | undefined; - const mailerConfig = mailer?.config as MailerConfig | undefined; - const databaseConfig = database?.config as DatabaseConfig | undefined; - - return { serverConfig, mailerConfig, databaseConfig }; -}; diff --git a/packages/frontend/admin/src/modules/nav/collapsible-item.tsx b/packages/frontend/admin/src/modules/nav/collapsible-item.tsx index 678c710a55..ab7dbada04 100644 --- a/packages/frontend/admin/src/modules/nav/collapsible-item.tsx +++ b/packages/frontend/admin/src/modules/nav/collapsible-item.tsx @@ -5,20 +5,15 @@ import { AccordionTrigger, } from '@affine/admin/components/ui/accordion'; import { useCallback } from 'react'; -import { NavLink, useLocation } from 'react-router-dom'; +import { NavLink } from 'react-router-dom'; export const CollapsibleItem = ({ - items, title, changeModule, }: { title: string; - items: string[]; changeModule?: (module: string) => void; }) => { - const location = useLocation(); - const activeSubTab = location.hash.slice(1); - const handleClick = useCallback( (id: string) => { const targetElement = document.getElementById(id); @@ -50,26 +45,6 @@ export const CollapsibleItem = ({ {title} - - {items.map(item => ( - { - return isActive && activeSubTab === item - ? `transition-all overflow-hidden w-full bg-zinc-100 inline-flex items-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50` - : ''; - }} - > - handleClick(item)} - className={`py-1 px-2 rounded text-ellipsis whitespace-nowrap overflow-hidden`} - > - {item} - - - ))} - ); @@ -79,10 +54,7 @@ export const OtherModules = ({ moduleList, changeModule, }: { - moduleList: { - moduleName: string; - keys: string[]; - }[]; + moduleList: string[]; changeModule?: (module: string) => void; }) => { return ( @@ -94,9 +66,8 @@ export const OtherModules = ({ {moduleList.map(module => ( ))} diff --git a/packages/frontend/admin/src/modules/nav/nav.tsx b/packages/frontend/admin/src/modules/nav/nav.tsx index 2f9224689d..5adfe7b8bf 100644 --- a/packages/frontend/admin/src/modules/nav/nav.tsx +++ b/packages/frontend/admin/src/modules/nav/nav.tsx @@ -84,12 +84,6 @@ export function Nav({ isCollapsed = false }: NavProps) { isCollapsed && 'items-center px-0 gap-1 overflow-visible' )} > - } - label="Server" - isCollapsed={isCollapsed} - /> } @@ -102,8 +96,13 @@ export function Nav({ isCollapsed = false }: NavProps) { label="AI" isCollapsed={isCollapsed} /> - + } + label="Server" + isCollapsed={isCollapsed} + />
{ const version = serverConfig?.version; const handleClick = useCallback(() => { - window.open(availableUpgrade.url, '_blank'); + if (availableUpgrade) { + window.open(availableUpgrade.url, '_blank'); + } }, [availableUpgrade]); if (availableUpgrade) { diff --git a/packages/frontend/admin/src/modules/nav/settings-item.tsx b/packages/frontend/admin/src/modules/nav/settings-item.tsx index e9ba5b31dc..8a1c26f596 100644 --- a/packages/frontend/admin/src/modules/nav/settings-item.tsx +++ b/packages/frontend/admin/src/modules/nav/settings-item.tsx @@ -10,23 +10,19 @@ import { SettingsIcon } from '@blocksuite/icons/rc'; import * as NavigationMenuPrimitive from '@radix-ui/react-navigation-menu'; import * as ScrollAreaPrimitive from '@radix-ui/react-scroll-area'; import { cssVarV2 } from '@toeverything/theme/v2'; -import { useMemo } from 'react'; import { NavLink } from 'react-router-dom'; -import { useGetServerRuntimeConfig } from '../settings/use-get-server-runtime-config'; +import { ALL_CONFIGURABLE_MODULES } from '../settings/config'; import { CollapsibleItem, OtherModules } from './collapsible-item'; import { useNav } from './context'; -export const SettingsItem = ({ isCollapsed }: { isCollapsed: boolean }) => { - const { moduleList } = useGetServerRuntimeConfig(); - const { setCurrentModule } = useNav(); - const { authModule, otherModules } = useMemo(() => { - const authModule = moduleList.find(module => module.moduleName === 'auth'); - const otherModules = moduleList.filter( - module => module.moduleName !== 'auth' - ); - return { authModule, otherModules }; - }, [moduleList]); +const authModule = ALL_CONFIGURABLE_MODULES.find(module => module === 'auth'); +const otherModules = ALL_CONFIGURABLE_MODULES.filter( + module => module !== 'auth' +); + +export const SettingsItem = ({ isCollapsed }: { isCollapsed: boolean }) => { + const { setCurrentModule } = useNav(); if (isCollapsed) { return ( @@ -63,10 +59,10 @@ export const SettingsItem = ({ isCollapsed }: { isCollapsed: boolean }) => { borderColor: cssVarV2('layer/insideBorder/blackBorder'), }} > - {moduleList.map(module => ( -
  • + {ALL_CONFIGURABLE_MODULES.map(module => ( +
  • { ? cssVarV2('selfhost/button/sidebarButton/bg/select') : undefined, })} - onClick={() => setCurrentModule?.(module.moduleName)} + onClick={() => setCurrentModule?.(module)} > - {module.moduleName} + {module}
  • ))} @@ -93,6 +89,7 @@ export const SettingsItem = ({ isCollapsed }: { isCollapsed: boolean }) => { ); } + return ( { {authModule && ( )} diff --git a/packages/frontend/admin/src/modules/settings/config-input.tsx b/packages/frontend/admin/src/modules/settings/config-input.tsx new file mode 100644 index 0000000000..e54144ebc9 --- /dev/null +++ b/packages/frontend/admin/src/modules/settings/config-input.tsx @@ -0,0 +1,125 @@ +import { Input } from '@affine/admin/components/ui/input'; +import { Switch } from '@affine/admin/components/ui/switch'; +import { useCallback, useState } from 'react'; + +import { isEqual } from './utils'; + +interface ConfigInputProps { + module: string; + field: string; + type: string; + defaultValue: any; + onChange: (module: string, field: string, value: any) => void; +} + +const Inputs: Record< + string, + React.ComponentType<{ + defaultValue: any; + onChange: (value?: any) => void; + }> +> = { + Boolean: function SwitchInput({ defaultValue, onChange }) { + const handleSwitchChange = (checked: boolean) => { + onChange(checked); + }; + + return ( + + ); + }, + String: function StringInput({ defaultValue, onChange }) { + const handleInputChange = (e: React.ChangeEvent) => { + onChange(e.target.value); + }; + + return ( + + ); + }, + Number: function NumberInput({ defaultValue, onChange }) { + const handleInputChange = (e: React.ChangeEvent) => { + onChange(parseInt(e.target.value)); + }; + + return ( + + ); + }, + JSON: function ObjectInput({ defaultValue, onChange }) { + const handleInputChange = (e: React.ChangeEvent) => { + try { + const value = JSON.parse(e.target.value); + onChange(value); + } catch {} + }; + + return ( + + ); + }, +}; + +export const ConfigInput = ({ + module, + field, + type, + defaultValue, + onChange, +}: ConfigInputProps) => { + const [value, setValue] = useState(defaultValue); + + const onValueChange = useCallback( + (value?: any) => { + onChange(module, field, value); + setValue(value); + }, + [module, field, onChange] + ); + + const Input = Inputs[type] ?? Inputs.JSON; + + const isValueEqual = isEqual(value, defaultValue); + + return ( +
    + +
    + + {JSON.stringify(defaultValue)} + {' '} + =>{' '} + + {JSON.stringify(value)} + +
    +
    + ); +}; diff --git a/packages/frontend/admin/src/modules/settings/config.ts b/packages/frontend/admin/src/modules/settings/config.ts new file mode 100644 index 0000000000..de67cd6b56 --- /dev/null +++ b/packages/frontend/admin/src/modules/settings/config.ts @@ -0,0 +1,31 @@ +import CONFIG from '../../config.json'; + +export type ConfigDescriptor = { + desc: string; + type: 'String' | 'Number' | 'Boolean' | 'Array' | 'Object'; + env?: string; + link?: string; +}; + +export type AppConfig = typeof CONFIG; +export type AvailableConfig = { + [K in keyof AppConfig]: { + module: K; + fields: Array; + }; +}[keyof AppConfig]; + +const IGNORED_MODULES: (keyof AppConfig)[] = [ + 'db', + 'redis', + 'copilot', // not ready +]; + +if (!environment.isSelfHosted) { + IGNORED_MODULES.push('payment'); +} + +export { CONFIG as ALL_CONFIG }; +export const ALL_CONFIGURABLE_MODULES = Object.keys(CONFIG).filter( + key => !IGNORED_MODULES.includes(key as keyof AppConfig) +) as (keyof AppConfig)[]; diff --git a/packages/frontend/admin/src/modules/settings/confirm-changes.tsx b/packages/frontend/admin/src/modules/settings/confirm-changes.tsx index 4bead3d04a..b4fa3993e1 100644 --- a/packages/frontend/admin/src/modules/settings/confirm-changes.tsx +++ b/packages/frontend/admin/src/modules/settings/confirm-changes.tsx @@ -7,22 +7,27 @@ import { DialogHeader, DialogTitle, } from '@affine/admin/components/ui/dialog'; - -import type { ModifiedValues } from './index'; +import { useCallback } from 'react'; export const ConfirmChanges = ({ + updates, open, - onClose, - onConfirm, onOpenChange, - modifiedValues, + onConfirm, }: { + updates: Record; open: boolean; - onClose: () => void; - onConfirm: () => void; onOpenChange: (open: boolean) => void; - modifiedValues: ModifiedValues[]; + onConfirm: () => void; }) => { + const onClose = useCallback(() => { + onOpenChange(false); + }, [onOpenChange]); + + const modifiedKeys = Object.keys(updates).filter( + key => updates[key].from !== updates[key].to + ); + return ( @@ -34,12 +39,12 @@ export const ConfirmChanges = ({ Are you sure you want to save the following changes? - {modifiedValues.length > 0 ? ( + {modifiedKeys.length > 0 ? (
                 

    {'{'}

    - {modifiedValues.map(({ id, expiredValue, newValue }) => ( -

    - {' '} {id}:{' '} + {modifiedKeys.map(key => ( +

    + {' '} {key}:{' '} - {JSON.stringify(expiredValue)} + {JSON.stringify(updates[key].from)} - {JSON.stringify(newValue)} + {JSON.stringify(updates[key].to)} ,

    @@ -70,7 +75,11 @@ export const ConfirmChanges = ({ -
    diff --git a/packages/frontend/admin/src/modules/settings/index.tsx b/packages/frontend/admin/src/modules/settings/index.tsx index ea52443010..8d2c2c5661 100644 --- a/packages/frontend/admin/src/modules/settings/index.tsx +++ b/packages/frontend/admin/src/modules/settings/index.tsx @@ -1,75 +1,47 @@ import { Button } from '@affine/admin/components/ui/button'; import { ScrollArea } from '@affine/admin/components/ui/scroll-area'; import { Separator } from '@affine/admin/components/ui/separator'; -import type { RuntimeConfigType } from '@affine/graphql'; +import { get } from 'lodash-es'; import { CheckIcon } from 'lucide-react'; -import type { Dispatch, SetStateAction } from 'react'; -import { useCallback, useMemo, useState } from 'react'; +import { useCallback, useState } from 'react'; import { Header } from '../header'; import { useNav } from '../nav/context'; +import { + ALL_CONFIG, + ALL_CONFIGURABLE_MODULES, + type ConfigDescriptor, +} from './config'; +import { ConfigInput } from './config-input'; import { ConfirmChanges } from './confirm-changes'; import { RuntimeSettingRow } from './runtime-setting-row'; -import { useGetServerRuntimeConfig } from './use-get-server-runtime-config'; -import { useUpdateServerRuntimeConfigs } from './use-update-server-runtime-config'; -import { - formatValue, - formatValueForInput, - isEqual, - renderInput, -} from './utils'; - -export type ModifiedValues = { - id: string; - expiredValue: any; - newValue: any; -}; +import { useAppConfig } from './use-app-config'; export function SettingsPage() { - const { trigger } = useUpdateServerRuntimeConfigs(); - const { serverRuntimeConfig } = useGetServerRuntimeConfig(); + const { appConfig, update, save, updates } = useAppConfig(); const [open, setOpen] = useState(false); - const [configValues, setConfigValues] = useState( - serverRuntimeConfig.reduce( - (acc, config) => { - acc[config.id] = config.value; - return acc; - }, - {} as Record - ) - ); - const modifiedValues: ModifiedValues[] = useMemo(() => { - return serverRuntimeConfig - .filter(config => !isEqual(config.value, configValues[config.id])) - .map(config => ({ - id: config.id, - key: config.key, - expiredValue: config.value, - newValue: configValues[config.id], - })); - }, [configValues, serverRuntimeConfig]); - const handleSave = useCallback(() => { - // post value example: { "key1": "newValue1","key2": "newValue2"} - const updates: Record = {}; - - modifiedValues.forEach(item => { - if (item.id && item.newValue !== undefined) { - updates[item.id] = item.newValue; - } - }); - trigger({ updates }); - }, [modifiedValues, trigger]); - - const disableSave = modifiedValues.length === 0; const onOpen = useCallback(() => setOpen(true), [setOpen]); - const onClose = useCallback(() => setOpen(false), [setOpen]); - const onConfirm = useCallback(() => { + + const disableSave = Object.keys(updates).length === 0; + + const saveChanges = useCallback(() => { if (disableSave) { return; } - handleSave(); - onClose(); - }, [disableSave, handleSave, onClose]); + save( + Object.entries(updates).map(([key, { to }]) => { + const splitAt = key.indexOf('.'); + const [module, field] = [key.slice(0, splitAt), key.slice(splitAt + 1)]; + return { + module, + key: field, + value: to, + }; + }) + ); + setOpen(false); + }, [save, disableSave, updates]); + return (
    } /> - +
    ); } export const AdminPanel = ({ - setConfigValues, - configValues, + appConfig, + onUpdate, }: { - setConfigValues: Dispatch>>; - configValues: Record; + appConfig: Record; + onUpdate: (module: string, field: string, value: any) => void; }) => { - const { configGroup } = useGetServerRuntimeConfig(); - const { currentModule } = useNav(); - const handleInputChange = useCallback( - (key: string, value: any, type: RuntimeConfigType) => { - const newValue = formatValueForInput(value, type); - setConfigValues(prevValues => ({ - ...prevValues, - [key]: newValue, - })); - }, - [setConfigValues] - ); - return (
    - {configGroup - .filter(group => group.moduleName === currentModule) - .map(group => { - const { moduleName, configs } = group; - return ( -
    -
    {moduleName}
    - {configs?.map((config, index) => { - const { id, type, description, updatedAt } = config; - const isValueEqual = isEqual(config.value, configValues[id]); - const formatServerValue = formatValue(config.value); - const formatCurrentValue = formatValue(configValues[id]); - return ( -
    - {index !== 0 && } - - handleInputChange(id, value, type) - )} - > -
    - - {formatServerValue} - {' '} - =>{' '} - - {formatCurrentValue} - -
    -
    -
    - ); - })} -
    - ); - })} + {ALL_CONFIGURABLE_MODULES.filter( + module => module === currentModule + ).map(module => { + const fields = Object.keys(ALL_CONFIG[module]); + return ( +
    +
    {module}
    + {fields.map((field, index) => { + // @ts-expect-error allow + const { desc, type } = ALL_CONFIG[module][ + field + ] as ConfigDescriptor; + + return ( +
    + {index !== 0 && } + + + +
    + ); + })} +
    + ); + })}
    ); diff --git a/packages/frontend/admin/src/modules/settings/runtime-setting-row.tsx b/packages/frontend/admin/src/modules/settings/runtime-setting-row.tsx index db55194de1..4c6377aeab 100644 --- a/packages/frontend/admin/src/modules/settings/runtime-setting-row.tsx +++ b/packages/frontend/admin/src/modules/settings/runtime-setting-row.tsx @@ -3,20 +3,15 @@ import { type ReactNode } from 'react'; export const RuntimeSettingRow = ({ id, description, - lastUpdatedTime, - operation, children, }: { id: string; description: string; - lastUpdatedTime: string; - operation: ReactNode; children: ReactNode; }) => { - const formatTime = new Date(lastUpdatedTime).toLocaleString(); return (
    @@ -26,14 +21,8 @@ export const RuntimeSettingRow = ({ {id}
    -
    - last updated at: {formatTime} -
    -
    -
    - {operation} - {children}
    +
    {children}
    ); }; diff --git a/packages/frontend/admin/src/modules/settings/use-app-config.ts b/packages/frontend/admin/src/modules/settings/use-app-config.ts new file mode 100644 index 0000000000..a0f31d3c7f --- /dev/null +++ b/packages/frontend/admin/src/modules/settings/use-app-config.ts @@ -0,0 +1,75 @@ +import { useMutation } from '@affine/admin/use-mutation'; +import { useQuery } from '@affine/admin/use-query'; +import { notify } from '@affine/component'; +import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks'; +import { UserFriendlyError } from '@affine/error'; +import { + appConfigQuery, + type UpdateAppConfigInput, + updateAppConfigMutation, +} from '@affine/graphql'; +import { get, merge } from 'lodash-es'; +import { useCallback, useState } from 'react'; + +export { type UpdateAppConfigInput }; + +export const useAppConfig = () => { + const { + data: { appConfig }, + mutate, + } = useQuery({ + query: appConfigQuery, + }); + + const { trigger } = useMutation({ + mutation: updateAppConfigMutation, + }); + + const [updates, setUpdates] = useState< + Record + >({}); + + const save = useAsyncCallback( + async (updates: UpdateAppConfigInput[]) => { + try { + const savedUpdates = await trigger({ + updates, + }); + await mutate({ appConfig: merge({}, appConfig, savedUpdates) }); + setUpdates({}); + notify.success({ + title: 'Saved successfully', + message: 'Runtime configurations have been saved successfully.', + }); + } catch (e) { + const error = UserFriendlyError.fromAny(e); + notify.error({ + title: 'Failed to save', + message: error.message, + }); + console.error(e); + } + }, + [appConfig, mutate, trigger] + ); + + const update = useCallback( + (module: string, field: string, value: any) => { + setUpdates(prev => ({ + ...prev, + [`${module}.${field}`]: { + from: get(appConfig, `${module}.${field}`), + to: value, + }, + })); + }, + [appConfig] + ); + + return { + appConfig, + update, + save, + updates, + }; +}; diff --git a/packages/frontend/admin/src/modules/settings/use-get-server-runtime-config.ts b/packages/frontend/admin/src/modules/settings/use-get-server-runtime-config.ts deleted file mode 100644 index cc0988d399..0000000000 --- a/packages/frontend/admin/src/modules/settings/use-get-server-runtime-config.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { useQuery } from '@affine/admin/use-query'; -import { getServerRuntimeConfigQuery } from '@affine/graphql'; -import { useMemo } from 'react'; - -export const useGetServerRuntimeConfig = () => { - const { data } = useQuery({ - query: getServerRuntimeConfigQuery, - }); - - const serverRuntimeConfig = useMemo( - () => - data?.serverRuntimeConfig.sort((a, b) => a.id.localeCompare(b.id)) ?? [], - [data] - ); - - // collect all the modules and config keys in each module - const moduleList = useMemo(() => { - const moduleMap: { [key: string]: string[] } = {}; - - serverRuntimeConfig.forEach(config => { - if (!moduleMap[config.module]) { - moduleMap[config.module] = []; - } - moduleMap[config.module].push(config.key); - }); - - return Object.keys(moduleMap) - .sort((a, b) => a.localeCompare(b)) - .map(moduleName => ({ - moduleName, - keys: moduleMap[moduleName].sort((a, b) => a.localeCompare(b)), - })); - }, [serverRuntimeConfig]); - - // group config by module name - const configGroup = useMemo(() => { - const configMap = new Map(); - - serverRuntimeConfig.forEach(config => { - if (!configMap.has(config.module)) { - configMap.set(config.module, []); - } - configMap.get(config.module)?.push(config); - }); - - return Array.from(configMap.entries()).map(([moduleName, configs]) => ({ - moduleName, - configs, - })); - }, [serverRuntimeConfig]); - - return { - serverRuntimeConfig, - moduleList, - configGroup, - }; -}; diff --git a/packages/frontend/admin/src/modules/settings/use-update-server-runtime-config.ts b/packages/frontend/admin/src/modules/settings/use-update-server-runtime-config.ts deleted file mode 100644 index 7c10f92150..0000000000 --- a/packages/frontend/admin/src/modules/settings/use-update-server-runtime-config.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { - useMutateQueryResource, - useMutation, -} from '@affine/admin/use-mutation'; -import { notify } from '@affine/component'; -import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks'; -import { - getServerRuntimeConfigQuery, - updateServerRuntimeConfigsMutation, -} from '@affine/graphql'; - -export const useUpdateServerRuntimeConfigs = () => { - const { trigger, isMutating } = useMutation({ - mutation: updateServerRuntimeConfigsMutation, - }); - const revalidate = useMutateQueryResource(); - - return { - trigger: useAsyncCallback( - async (values: any) => { - try { - await trigger(values); - await revalidate(getServerRuntimeConfigQuery); - notify.success({ - title: 'Saved successfully', - message: 'Runtime configurations have been saved successfully.', - }); - } catch (e) { - notify.error({ - title: 'Failed to save', - message: - 'Failed to save runtime configurations, please try again later.', - }); - console.error(e); - } - }, - [revalidate, trigger] - ), - isMutating, - }; -}; diff --git a/packages/frontend/admin/src/modules/settings/utils.tsx b/packages/frontend/admin/src/modules/settings/utils.tsx index fd69ef25ff..d54c84fb69 100644 --- a/packages/frontend/admin/src/modules/settings/utils.tsx +++ b/packages/frontend/admin/src/modules/settings/utils.tsx @@ -1,73 +1,5 @@ -import { Input } from '@affine/admin/components/ui/input'; -import { Switch } from '@affine/admin/components/ui/switch'; -import type { RuntimeConfigType } from '@affine/graphql'; - -export const renderInput = ( - type: RuntimeConfigType, - value: any, - onChange: (value?: any) => void -) => { - const handleInputChange = (e: React.ChangeEvent) => { - onChange(e.target.value); - }; - const handleSwitchChange = (checked: boolean) => { - onChange(checked); - }; - switch (type) { - case 'Boolean': - return ; - case 'String': - return ( - - ); - case 'Number': - return ( -
    - -
    - ); - // TODO(@JimmFly): add more types - default: - return null; - } -}; - export const isEqual = (a: any, b: any) => { if (typeof a !== typeof b) return false; if (typeof a === 'object') return JSON.stringify(a) === JSON.stringify(b); return a === b; }; - -export const formatValue = (value: any) => { - if (typeof value === 'object') return JSON.stringify(value); - return value.toString(); -}; - -export const formatValueForInput = (value: any, type: RuntimeConfigType) => { - let newValue = null; - switch (type) { - case 'Boolean': - newValue = !!value; - break; - case 'String': - newValue = value; - break; - case 'Number': - newValue = Number(value); - break; - case 'Array': - newValue = value.split(','); - break; - case 'Object': - newValue = JSON.parse(value); - break; - default: - break; - } - return newValue; -}; diff --git a/packages/frontend/admin/tsconfig.json b/packages/frontend/admin/tsconfig.json index d92e6b6788..85cab81c21 100644 --- a/packages/frontend/admin/tsconfig.json +++ b/packages/frontend/admin/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "../../../tsconfig.web.json", - "include": ["./src"], + "include": ["./src", "./src/config.json"], "compilerOptions": { + "resolveJsonModule": true, "rootDir": "./src", "outDir": "./dist", "tsBuildInfoFile": "./dist/tsconfig.tsbuildinfo" @@ -9,6 +10,7 @@ "references": [ { "path": "../component" }, { "path": "../core" }, + { "path": "../../common/error" }, { "path": "../../common/graphql" }, { "path": "../../common/infra" } ] diff --git a/packages/frontend/i18n/src/i18n.gen.ts b/packages/frontend/i18n/src/i18n.gen.ts index c58f82531c..cfce617cf3 100644 --- a/packages/frontend/i18n/src/i18n.gen.ts +++ b/packages/frontend/i18n/src/i18n.gen.ts @@ -8048,6 +8048,10 @@ export function useAFFiNEI18N(): { * `You can not mention yourself.` */ ["error.MENTION_USER_ONESELF_DENIED"](): string; + /** + * `Invalid app config.` + */ + ["error.INVALID_APP_CONFIG"](): string; } { const { t } = useTranslation(); return useMemo(() => createProxy((key) => t.bind(null, key)), [t]); } function createComponent(i18nKey: string) { return (props) => createElement(Trans, { i18nKey, shouldUnescape: true, ...props }); diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index e2f7a05880..66dcaa5b99 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -1985,5 +1985,6 @@ "error.UNSUPPORTED_CLIENT_VERSION": "Unsupported client with version [{{clientVersion}}], required version is [{{requiredVersion}}].", "error.NOTIFICATION_NOT_FOUND": "Notification not found.", "error.MENTION_USER_DOC_ACCESS_DENIED": "Mentioned user can not access doc {{docId}}.", - "error.MENTION_USER_ONESELF_DENIED": "You can not mention yourself." + "error.MENTION_USER_ONESELF_DENIED": "You can not mention yourself.", + "error.INVALID_APP_CONFIG": "Invalid app config." } diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 3cf134b578..d56a190407 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -908,6 +908,7 @@ export const PackageList = [ workspaceDependencies: [ 'packages/frontend/component', 'packages/frontend/core', + 'packages/common/error', 'packages/common/graphql', 'packages/common/infra', ], diff --git a/yarn.lock b/yarn.lock index 94c5fe4ad7..ec24728f06 100644 --- a/yarn.lock +++ b/yarn.lock @@ -173,6 +173,7 @@ __metadata: dependencies: "@affine/component": "workspace:*" "@affine/core": "workspace:*" + "@affine/error": "workspace:*" "@affine/graphql": "workspace:*" "@blocksuite/icons": "npm:^2.2.8" "@radix-ui/react-accordion": "npm:^1.2.2" @@ -206,12 +207,14 @@ __metadata: "@tanstack/react-table": "npm:^8.20.5" "@toeverything/infra": "workspace:*" "@toeverything/theme": "npm:^1.1.12" + "@types/lodash-es": "npm:^4.17.12" class-variance-authority: "npm:^0.7.1" clsx: "npm:^2.1.1" cmdk: "npm:^1.0.4" cross-env: "npm:^7.0.3" embla-carousel-react: "npm:^8.5.1" input-otp: "npm:^1.4.1" + lodash-es: "npm:^4.17.21" lucide-react: "npm:^0.484.0" next-themes: "npm:^0.4.4" react: "npm:^19.0.0" @@ -960,6 +963,7 @@ __metadata: tldts: "npm:^6.1.68" ts-node: "npm:^10.9.2" typescript: "npm:^5.7.2" + why-is-node-running: "npm:^3.2.2" winston: "npm:^3.17.0" yjs: "npm:^13.6.21" zod: "npm:^3.24.1" @@ -34116,6 +34120,15 @@ __metadata: languageName: node linkType: hard +"why-is-node-running@npm:^3.2.2": + version: 3.2.2 + resolution: "why-is-node-running@npm:3.2.2" + bin: + why-is-node-running: cli.js + checksum: 10/b57146897f676cf01fe30ac415f66dd72b54218e6fee8eb4479251f72a2e064bc18ed7eaddac18513f978353ce2b758d16a4de126cd3e47e467631c1ec5a50af + languageName: node + linkType: hard + "wide-align@npm:^1.1.5": version: 1.1.5 resolution: "wide-align@npm:1.1.5"